{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic回归 -- Renting Listing Inqueries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import必要的模块\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# 用于计算feature字段的文本特征提取\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "# CountVectorizer为稀疏特征，特征编码结果存为稀疏矩阵xgboost处理更高效\n",
    "from scipy import sparse\n",
    "\n",
    "#对类别特征进行编码\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# 对地理位置通过聚类进行离散化\n",
    "from sklearn.cluster import KMeans\n",
    "from nltk.metrics import distance as distance\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 读取数据 & 数据探索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>building_id</th>\n",
       "      <th>created</th>\n",
       "      <th>description</th>\n",
       "      <th>display_address</th>\n",
       "      <th>features</th>\n",
       "      <th>interest_level</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>manager_id</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>street_address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>53a5b119ba8f7b61d4e010512e0dfc85</td>\n",
       "      <td>2016-06-24 07:54:24</td>\n",
       "      <td>A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...</td>\n",
       "      <td>Metropolitan Avenue</td>\n",
       "      <td>[]</td>\n",
       "      <td>medium</td>\n",
       "      <td>40.7145</td>\n",
       "      <td>7211212</td>\n",
       "      <td>-73.9425</td>\n",
       "      <td>5ba989232d0489da1b5f2c45f6688adc</td>\n",
       "      <td>[https://photos.renthop.com/2/7211212_1ed4542e...</td>\n",
       "      <td>3000</td>\n",
       "      <td>792 Metropolitan Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10000</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>c5c8a357cba207596b04d1afd1e4f130</td>\n",
       "      <td>2016-06-12 12:19:27</td>\n",
       "      <td></td>\n",
       "      <td>Columbus Avenue</td>\n",
       "      <td>[Doorman, Elevator, Fitness Center, Cats Allow...</td>\n",
       "      <td>low</td>\n",
       "      <td>40.7947</td>\n",
       "      <td>7150865</td>\n",
       "      <td>-73.9667</td>\n",
       "      <td>7533621a882f71e25173b27e3139d83d</td>\n",
       "      <td>[https://photos.renthop.com/2/7150865_be3306c5...</td>\n",
       "      <td>5465</td>\n",
       "      <td>808 Columbus Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100004</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>c3ba40552e2120b0acfc3cb5730bb2aa</td>\n",
       "      <td>2016-04-17 03:26:41</td>\n",
       "      <td>Top Top West Village location, beautiful Pre-w...</td>\n",
       "      <td>W 13 Street</td>\n",
       "      <td>[Laundry In Building, Dishwasher, Hardwood Flo...</td>\n",
       "      <td>high</td>\n",
       "      <td>40.7388</td>\n",
       "      <td>6887163</td>\n",
       "      <td>-74.0018</td>\n",
       "      <td>d9039c43983f6e564b1482b273bd7b01</td>\n",
       "      <td>[https://photos.renthop.com/2/6887163_de85c427...</td>\n",
       "      <td>2850</td>\n",
       "      <td>241 W 13 Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100007</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>28d9ad350afeaab8027513a3e52ac8d5</td>\n",
       "      <td>2016-04-18 02:22:02</td>\n",
       "      <td>Building Amenities - Garage - Garden - fitness...</td>\n",
       "      <td>East 49th Street</td>\n",
       "      <td>[Hardwood Floors, No Fee]</td>\n",
       "      <td>low</td>\n",
       "      <td>40.7539</td>\n",
       "      <td>6888711</td>\n",
       "      <td>-73.9677</td>\n",
       "      <td>1067e078446a7897d2da493d2f741316</td>\n",
       "      <td>[https://photos.renthop.com/2/6888711_6e660cee...</td>\n",
       "      <td>3275</td>\n",
       "      <td>333 East 49th Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100013</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-04-28 01:32:41</td>\n",
       "      <td>Beautifully renovated 3 bedroom flex 4 bedroom...</td>\n",
       "      <td>West 143rd Street</td>\n",
       "      <td>[Pre-War]</td>\n",
       "      <td>low</td>\n",
       "      <td>40.8241</td>\n",
       "      <td>6934781</td>\n",
       "      <td>-73.9493</td>\n",
       "      <td>98e13ad4b495b9613cef886d79a6291f</td>\n",
       "      <td>[https://photos.renthop.com/2/6934781_1fa4b41a...</td>\n",
       "      <td>3350</td>\n",
       "      <td>500 West 143rd Street</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        bathrooms  bedrooms                       building_id  \\\n",
       "10            1.5         3  53a5b119ba8f7b61d4e010512e0dfc85   \n",
       "10000         1.0         2  c5c8a357cba207596b04d1afd1e4f130   \n",
       "100004        1.0         1  c3ba40552e2120b0acfc3cb5730bb2aa   \n",
       "100007        1.0         1  28d9ad350afeaab8027513a3e52ac8d5   \n",
       "100013        1.0         4                                 0   \n",
       "\n",
       "                    created  \\\n",
       "10      2016-06-24 07:54:24   \n",
       "10000   2016-06-12 12:19:27   \n",
       "100004  2016-04-17 03:26:41   \n",
       "100007  2016-04-18 02:22:02   \n",
       "100013  2016-04-28 01:32:41   \n",
       "\n",
       "                                              description  \\\n",
       "10      A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...   \n",
       "10000                                                       \n",
       "100004  Top Top West Village location, beautiful Pre-w...   \n",
       "100007  Building Amenities - Garage - Garden - fitness...   \n",
       "100013  Beautifully renovated 3 bedroom flex 4 bedroom...   \n",
       "\n",
       "            display_address  \\\n",
       "10      Metropolitan Avenue   \n",
       "10000       Columbus Avenue   \n",
       "100004          W 13 Street   \n",
       "100007     East 49th Street   \n",
       "100013    West 143rd Street   \n",
       "\n",
       "                                                 features interest_level  \\\n",
       "10                                                     []         medium   \n",
       "10000   [Doorman, Elevator, Fitness Center, Cats Allow...            low   \n",
       "100004  [Laundry In Building, Dishwasher, Hardwood Flo...           high   \n",
       "100007                          [Hardwood Floors, No Fee]            low   \n",
       "100013                                          [Pre-War]            low   \n",
       "\n",
       "        latitude  listing_id  longitude                        manager_id  \\\n",
       "10       40.7145     7211212   -73.9425  5ba989232d0489da1b5f2c45f6688adc   \n",
       "10000    40.7947     7150865   -73.9667  7533621a882f71e25173b27e3139d83d   \n",
       "100004   40.7388     6887163   -74.0018  d9039c43983f6e564b1482b273bd7b01   \n",
       "100007   40.7539     6888711   -73.9677  1067e078446a7897d2da493d2f741316   \n",
       "100013   40.8241     6934781   -73.9493  98e13ad4b495b9613cef886d79a6291f   \n",
       "\n",
       "                                                   photos  price  \\\n",
       "10      [https://photos.renthop.com/2/7211212_1ed4542e...   3000   \n",
       "10000   [https://photos.renthop.com/2/7150865_be3306c5...   5465   \n",
       "100004  [https://photos.renthop.com/2/6887163_de85c427...   2850   \n",
       "100007  [https://photos.renthop.com/2/6888711_6e660cee...   3275   \n",
       "100013  [https://photos.renthop.com/2/6934781_1fa4b41a...   3350   \n",
       "\n",
       "                 street_address  \n",
       "10      792 Metropolitan Avenue  \n",
       "10000       808 Columbus Avenue  \n",
       "100004          241 W 13 Street  \n",
       "100007     333 East 49th Street  \n",
       "100013    500 West 143rd Street  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读取数据\n",
    "train = pd.read_json('./AIData/RentListingInquries_train.json')\n",
    "test = pd.read_json('./AIData/RentListingInquries_test.json')\n",
    "\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 49352 entries, 10 to 99994\n",
      "Data columns (total 15 columns):\n",
      "bathrooms          49352 non-null float64\n",
      "bedrooms           49352 non-null int64\n",
      "building_id        49352 non-null object\n",
      "created            49352 non-null object\n",
      "description        49352 non-null object\n",
      "display_address    49352 non-null object\n",
      "features           49352 non-null object\n",
      "interest_level     49352 non-null object\n",
      "latitude           49352 non-null float64\n",
      "listing_id         49352 non-null int64\n",
      "longitude          49352 non-null float64\n",
      "manager_id         49352 non-null object\n",
      "photos             49352 non-null object\n",
      "price              49352 non-null int64\n",
      "street_address     49352 non-null object\n",
      "dtypes: float64(3), int64(3), object(9)\n",
      "memory usage: 6.0+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 74659 entries, 0 to 99999\n",
      "Data columns (total 14 columns):\n",
      "bathrooms          74659 non-null float64\n",
      "bedrooms           74659 non-null int64\n",
      "building_id        74659 non-null object\n",
      "created            74659 non-null object\n",
      "description        74659 non-null object\n",
      "display_address    74659 non-null object\n",
      "features           74659 non-null object\n",
      "latitude           74659 non-null float64\n",
      "listing_id         74659 non-null int64\n",
      "longitude          74659 non-null float64\n",
      "manager_id         74659 non-null object\n",
      "photos             74659 non-null object\n",
      "price              74659 non-null int64\n",
      "street_address     74659 non-null object\n",
      "dtypes: float64(3), int64(3), object(8)\n",
      "memory usage: 8.5+ MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>49352.00000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>4.935200e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.21218</td>\n",
       "      <td>1.541640</td>\n",
       "      <td>40.741545</td>\n",
       "      <td>7.024055e+06</td>\n",
       "      <td>-73.955716</td>\n",
       "      <td>3.830174e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.50142</td>\n",
       "      <td>1.115018</td>\n",
       "      <td>0.638535</td>\n",
       "      <td>1.262746e+05</td>\n",
       "      <td>1.177912</td>\n",
       "      <td>2.206687e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.811957e+06</td>\n",
       "      <td>-118.271000</td>\n",
       "      <td>4.300000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.728300</td>\n",
       "      <td>6.915888e+06</td>\n",
       "      <td>-73.991700</td>\n",
       "      <td>2.500000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.751800</td>\n",
       "      <td>7.021070e+06</td>\n",
       "      <td>-73.977900</td>\n",
       "      <td>3.150000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>40.774300</td>\n",
       "      <td>7.128733e+06</td>\n",
       "      <td>-73.954800</td>\n",
       "      <td>4.100000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>10.00000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>44.883500</td>\n",
       "      <td>7.753784e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.490000e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         bathrooms      bedrooms      latitude    listing_id     longitude  \\\n",
       "count  49352.00000  49352.000000  49352.000000  4.935200e+04  49352.000000   \n",
       "mean       1.21218      1.541640     40.741545  7.024055e+06    -73.955716   \n",
       "std        0.50142      1.115018      0.638535  1.262746e+05      1.177912   \n",
       "min        0.00000      0.000000      0.000000  6.811957e+06   -118.271000   \n",
       "25%        1.00000      1.000000     40.728300  6.915888e+06    -73.991700   \n",
       "50%        1.00000      1.000000     40.751800  7.021070e+06    -73.977900   \n",
       "75%        1.00000      2.000000     40.774300  7.128733e+06    -73.954800   \n",
       "max       10.00000      8.000000     44.883500  7.753784e+06      0.000000   \n",
       "\n",
       "              price  \n",
       "count  4.935200e+04  \n",
       "mean   3.830174e+03  \n",
       "std    2.206687e+04  \n",
       "min    4.300000e+01  \n",
       "25%    2.500000e+03  \n",
       "50%    3.150000e+03  \n",
       "75%    4.100000e+03  \n",
       "max    4.490000e+06  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0,0.5,u'Number of occurrences')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAHPVJREFUeJzt3XuUXWWd5vHvQ7iKlwQpWDGBDmp6\nFG9RIkSxbUUmBOwmSIPCMJJGllEHFJaXFuweolzstm2lmx5hGsdIsB0CIkjUYIxIQFAgAcIlXBYx\noMSkIRCuMoYmPPPHfktOilNVO2GfOjnk+ax1Vu39O+/e53eoVfzy7vfd75ZtIiIimrBVtxOIiIgX\njxSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGpKhERERjUlQiIqIxKSoREdGYrbudwEjbeeed\nPWHChG6nERHRU2688caHbPcN126LKyoTJkxgyZIl3U4jIqKnSPpNnXa5/BUREY3pWFGRtL2kGyTd\nImmZpC+V+HmS7pW0tLwmlbgknSVpuaRbJb2t5VwzJN1TXjNa4ntJuq0cc5Ykder7RETE8Dp5+Wsd\nsJ/tJyVtA1wj6fLy3udsXzyg/YHAxPLaBzgH2EfSTsAsYDJg4EZJ82w/UtrMBK4D5gPTgMuJiIiu\n6FhPxZUny+425TXUOvvTgfPLcdcBoyWNBQ4AFtpeWwrJQmBaee/ltn/lav3+84FDOvV9IiJieB0d\nU5E0StJS4EGqwnB9eeuMconrTEnbldg44P6Ww1eW2FDxlW3i7fKYKWmJpCVr1qx5wd8rIiLa62hR\nsb3e9iRgPLC3pDcCJwOvA94O7AR8vjRvNx7iTYi3y+Nc25NtT+7rG3ZGXEREbKIRmf1l+1FgETDN\n9upyiWsd8G1g79JsJbBby2HjgVXDxMe3iUdERJd0cvZXn6TRZXsHYH/grjIWQpmpdQhwezlkHnB0\nmQU2BXjM9mpgATBV0hhJY4CpwILy3hOSppRzHQ1c1qnvExERw+vk7K+xwBxJo6iK10W2fyTp55L6\nqC5fLQU+XtrPBw4ClgNPAccA2F4r6TRgcWl3qu21ZfsTwHnADlSzvjLzKyKii1RNnNpyTJ482bmj\n/sXvt6e+qdspbBF2P+W2bqcQI0TSjbYnD9cud9RHRERjUlQiIqIxKSoREdGYFJWIiGhMikpERDQm\nRSUiIhqTohIREY1JUYmIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhER\njUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0pmNFRdL2km6QdIukZZK+VOJ7SLpe\n0j2SLpS0bYlvV/aXl/cntJzr5BK/W9IBLfFpJbZc0kmd+i4REVFPJ3sq64D9bL8FmARMkzQF+Apw\npu2JwCPAsaX9scAjtl8LnFnaIWlP4AjgDcA04GxJoySNAr4BHAjsCRxZ2kZERJd0rKi48mTZ3aa8\nDOwHXFzic4BDyvb0sk95/32SVOJzba+zfS+wHNi7vJbbXmH7aWBuaRsREV3S0TGV0qNYCjwILAR+\nDTxq+5nSZCUwrmyPA+4HKO8/BryyNT7gmMHiERHRJR0tKrbX254EjKfqWby+XbPyU4O8t7Hx55E0\nU9ISSUvWrFkzfOIREbFJRmT2l+1HgUXAFGC0pK3LW+OBVWV7JbAbQHn/FcDa1viAYwaLt/v8c21P\ntj25r6+via8UERFtdHL2V5+k0WV7B2B/4E7gSuCw0mwGcFnZnlf2Ke//3LZL/IgyO2wPYCJwA7AY\nmFhmk21LNZg/r1PfJyIihrf18E022VhgTpmltRVwke0fSboDmCvpdOBm4Ful/beA70haTtVDOQLA\n9jJJFwF3AM8Ax9leDyDpeGABMAqYbXtZB79PREQMo2NFxfatwFvbxFdQja8MjP8BOHyQc50BnNEm\nPh+Y/4KTjYiIRuSO+oiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHR\nmBSViIhoTIpKREQ0ZtiiImlHSVuV7T+VdLCkbTqfWkRE9Jo6PZWrge0ljQOuAI4BzutkUhER0Zvq\nFBXZfgo4FPhX2x+geiZ8RETEBmoVFUnvAI4CflxinVwyPyIielSdonIicDJwaXm2yaupHrQVERGx\ngWF7HLavAq6StGPZXwF8qtOJRURE76kz++sd5WmNd5b9t0g6u+OZRUREz6lz+eufgQOAhwFs3wK8\nu5NJRUREb6p186Pt+weE1ncgl4iI6HF1ZnHdL+mdgCVtSzWecmdn04qIiF5Up6fyceA4YBywEphU\n9iMiIjYwbFGx/ZDto2zvansX2//d9sPDHSdpN0lXSrpT0jJJJ5T4FyX9TtLS8jqo5ZiTJS2XdLek\nA1ri00psuaSTWuJ7SLpe0j2SLiw9qYiI6JI6s7/mSBrdsj9G0uwa534G+Izt1wNTgOMk9d+Jf6bt\nSeU1v5x3T+AI4A3ANOBsSaMkjQK+ARxIdSf/kS3n+Uo510TgEeDYGnlFRESH1Ln89Wbbj/bv2H4E\neOtwB9lebfumsv0E1TjMuCEOmQ7Mtb3O9r3AcmDv8lpue4Xtp4G5wHRJAvYDLi7HzwEOqfF9IiKi\nQ+oUla0kjenfkbQTG7lMi6QJVIXo+hI6XtKtkma3nHsc0DrLbGWJDRZ/JfCo7WcGxCMiokvqFJWv\nAb+UdJqk04BfAv9Y9wMkvRT4PnCi7ceBc4DXUA34ry7nB1Cbw70J8XY5zJS0RNKSNWvW1E09IiI2\nUp2B+vOBw4AHgAeBQ21/p87Jy3NXvg981/Yl5XwP2F5v+1ngm1SXt6DqaezWcvh4YNUQ8YeA0ZK2\nHhBv9x3OtT3Z9uS+vr46qUdExCao++THu4BLgMuAJyXtPtwBZczjW8Cdtr/eEh/b0uwDwO1lex5w\nhKTtJO0BTARuABYDE8tMr22pBvPn2TbVwpaHleNnlPwiIqJLhh0bkfRJYBZVT2U91WUnA28e5tB9\ngQ8Dt0laWmJfoJq9Namc4z7gYwBlBeSLgDuoZo4dZ3t9yeF4YAEwCphte1k53+eBuZJOB26mKmIR\nEdEldQbcTwD+S517U1rZvob24x7zhzjmDOCMNvH57Y4rKybvPTAeERHdUefy1/3AY51OJCIiel+d\nnsoKYJGkHwPr+oOt4yQRERFQr6j8try2La+IiIi26jz58UsAkna0/fvOpxQREb0qT36MiIjG5MmP\nERHRmDz5MSIiGpMnP0ZERGPy5MeIiGjMkD2V8oCsD9s+aoTyiYiIHjZkT6WsvTV9hHKJiIgeV2dM\n5VpJ/wu4EPjjfSr9T3WMiIjoV6eovLP8PLUlZqpH+UZERPzRcGMqWwHn2L5ohPKJiIgeNtyYyrPA\n8SOUS0RE9Lg6U4oXSvqspN0k7dT/6nhmERHRc+qMqXyk/Gy9N8XAq5tPJyIielmdVYr3GIlEIiKi\n99V5Rv3R7eK2z28+nYiI6GV1Ln+9vWV7e+B9wE1AikpERGygzuWvT7buS3oF8J2OZRQRET2r1tL3\nAzwFTByuUZktdqWkOyUtk3RCie8kaaGke8rPMSUuSWdJWi7pVklvaznXjNL+HkkzWuJ7SbqtHHOW\nJG3C94mIiIbUefLjDyXNK68fAXcDl9U49zPAZ2y/HpgCHCdpT+Ak4ArbE4Eryj7AgVTFaiIwEzin\nfP5OwCxgH2BvYFZ/ISptZrYcN61GXhER0SF1xlT+qWX7GeA3tlcOd5Dt1cDqsv2EpDupls+fDryn\nNJsDLAI+X+Ln2zZwnaTRksaWtgttrwWQtBCYJmkR8HLbvyrx84FDgMtrfKeIiOiAOkXlt8Bq238A\nkLSDpAm276v7IZImAG8Frgd2LQUH26sl7VKajQNanzC5ssSGiq9sE4+IiC6pM6byPeDZlv31JVaL\npJcC3wdOtP34UE3bxLwJ8XY5zJS0RNKSNWvWDJdyRERsojpFZWvbT/fvlO1t65xc0jZUBeW7ti8p\n4QfKZS3KzwdLfCWwW8vh44FVw8THt4k/j+1zbU+2Pbmvr69O6hERsQnqFJU1kg7u35E0HXhouIPK\nTKxvAXfa/nrLW/OA/hlcM3hu0H8ecHSZBTYFeKxcJlsATJU0pgzQTwUWlPeekDSlfNbR1JtAEBER\nHVJnTOXjwHfLg7qg6iG0vct+gH2BDwO3SVpaYl8A/gG4SNKxVOM1h5f35gMHAcuppi0fA2B7raTT\ngMWl3an9g/bAJ4DzgB2oBugzSB8R0UV1bn78NTCljI3I9hN1Tmz7GtqPe0B1V/7A9mbDRStb35sN\nzG4TXwK8sU4+ERHReXXuU/mypNG2nyxTg8dIOn0kkouIiN5SZ0zlQNuP9u/YfoTqMlVERMQG6hSV\nUZK269+RtAOw3RDtIyJiC1VnoP7fgSskfZvqPpCPUN0JHxERsYE6A/X/KOlWYP8SOs32gs6mFRER\nvahOTwXgZmAbqp7KzZ1LJyIielmd2V8fBG4ADgM+CFwv6bBOJxYREb2nTk/lb4G3234QQFIf8DPg\n4k4mFhERvafO7K+t+gtK8XDN4yIiYgtTp6fyE0kLgAvK/oeollSJiIjYQJ3ZX5+TdCjwLqplV861\nfWnHM4uIiJ5Ta/ZXWbb+kmEbRkTEFi1jIxER0ZgUlYiIaMygRUXSFeXnV0YunYiI6GVDjamMlfTn\nwMGS5jLg2Si2b+poZhER0XOGKiqnACdRPfv96wPeM7Bfp5KKiIjeNGhRsX0xcLGk/2n7tBHMKSIi\nelSd+1ROk3Qw8O4SWmT7R51NKyIielGdBSX/HjgBuKO8TiixiIiIDdS5+fH9wCTbzwJImkO1/P3J\nnUwsIiJ6T937VEa3bL+iE4lERETvq1NU/h64WdJ5pZdyI/Dl4Q6SNFvSg5Jub4l9UdLvJC0tr4Na\n3jtZ0nJJd0s6oCU+rcSWSzqpJb6HpOsl3SPpQknb1v3SERHRGcMWFdsXAFOo1v66BHiH7bk1zn0e\nMK1N/Ezbk8prPoCkPYEjgDeUY86WNErSKOAbwIHAnsCRpS3AV8q5JgKPAMfWyCkiIjqo1uUv26tt\nz7N9me3/qHnM1cDamnlMB+baXmf7XmA5sHd5Lbe9wvbTwFxguiRR3SfT/6CwOcAhNT8rIiI6pBtr\nfx0v6dZyeWxMiY0D7m9ps7LEBou/EnjU9jMD4m1JmilpiaQla9asaep7RETEACNdVM4BXgNMAlYD\nXytxtWnrTYi3Zftc25NtT+7r69u4jCMiorYhi4qkrVoH2l8o2w/YXl+mJ3+T6vIWVD2N3VqajgdW\nDRF/CBgtaesB8YiI6KIhi0r5n/8tknZv4sMkjW3Z/QDQX7DmAUdI2k7SHsBE4AZgMTCxzPTalmow\nf55tA1cCh5XjZwCXNZFjRERsujo3P44Flkm6Afh9f9D2wUMdJOkC4D3AzpJWArOA90iaRHWp6j7g\nY+VcyyRdRHXH/jPAcbbXl/McDywARgGzbS8rH/F5YK6k06luxvxWnS8cERGdU6eofGlTTmz7yDbh\nQf/Hb/sM4Iw28fnA/DbxFTx3+SwiIjYDdRaUvErSnwATbf9M0kuoeg0REREbqLOg5Eep7gf5txIa\nB/ygk0lFRERvqjOl+DhgX+BxANv3ALt0MqmIiOhNdYrKunI3OwBlGu+g94RERMSWq05RuUrSF4Ad\nJP1X4HvADzubVkRE9KI6ReUkYA1wG9UU4PnA33UyqYiI6E11Zn89W5a8v57qstfd5ebDiIiIDQxb\nVCS9H/jfwK+p1tzaQ9LHbF/e6eQiIqK31Ln58WvAe20vB5D0GuDHQIpKRERsoM6YyoP9BaVYATzY\noXwiIqKHDdpTkXRo2VwmaT5wEdWYyuFUCz1GRERsYKjLX3/Zsv0A8Odlew0w5vnNIyJiSzdoUbF9\nzEgmEhERva/O7K89gE8CE1rbD7f0fUREbHnqzP76AdWS9T8Enu1sOhER0cvqFJU/2D6r45lERETP\nq1NU/kXSLOCnwLr+oO2bOpZVRET0pDpF5U3Ah4H9eO7yl8v+i9penzu/2ym86N341aO7nUJENKhO\nUfkA8OrW5e8jIiLaqXNH/S3A6E4nEhERva9OT2VX4C5Ji9lwTCVTiiMiYgN1isqsTTmxpNnAX1Ct\nHfbGEtsJuJDqnpf7gA/afkSSgH8BDgKeAv66fyKApBk89/yW023PKfG9gPOAHaie8XJCluSPiOiu\nYS9/2b6q3avGuc8Dpg2InQRcYXsicEXZBzgQmFheM4Fz4I9FaBawD7A3MEtS/xIx55S2/ccN/KyI\niBhhwxYVSU9Iery8/iBpvaTHhzvO9tXA2gHh6cCcsj0HOKQlfr4r1wGjJY0FDgAW2l5r+xFgITCt\nvPdy278qvZPzW84VERFdUufJjy9r3Zd0CFWvYVPsant1Oe9qSbuU+Djg/pZ2K0tsqPjKNvG2JM2k\n6tWw++67b2LqERExnDqzvzZg+wc0f4+K2n3UJsTbsn2u7cm2J/f19W1iihERMZw6C0oe2rK7FTCZ\nIf4HPowHJI0tvZSxPPewr5XAbi3txgOrSvw9A+KLSnx8m/YREdFFdXoqf9nyOgB4gmoMZFPMA2aU\n7RnAZS3xo1WZAjxWLpMtAKZKGlMG6KcCC8p7T0iaUmaOHd1yroiI6JI6Yyqb9FwVSRdQ9TJ2lrSS\nahbXPwAXSToW+C3VUyShmhJ8ELCcakrxMeWz10o6jeeeNHmq7f7B/0/w3JTiy8srIiK6aKjHCZ8y\nxHG2fdpQJ7Z95CBvva/dyYDjBjnPbGB2m/gS4I1D5RARESNrqJ7K79vEdgSOBV4JDFlUIiJiyzPU\n44S/1r8t6WXACVSXpeYCXxvsuIiI2HINOaZS7mj/NHAU1c2Kbys3IUZERDzPUGMqXwUOBc4F3mT7\nyRHLKiIietJQU4o/A7yKajHHVS1LtTxRZ5mWiIjY8gw1prLRd9tHRMSWLYUjIiIak6ISERGNSVGJ\niIjGpKhERERjUlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYmIiMakqERERGNS\nVCIiojEpKhER0ZgUlYiIaExXioqk+yTdJmmppCUltpOkhZLuKT/HlLgknSVpuaRbJb2t5TwzSvt7\nJM3oxneJiIjndLOn8l7bk2xPLvsnAVfYnghcUfYBDgQmltdM4ByoihAwC9gH2BuY1V+IIiKiOzan\ny1/TgTllew5wSEv8fFeuA0ZLGgscACy0vdb2I8BCYNpIJx0REc/pVlEx8FNJN0qaWWK72l4NUH7u\nUuLjgPtbjl1ZYoPFIyKiSwZ9Rn2H7Wt7laRdgIWS7hqirdrEPET8+SeoCtdMgN13331jc42IiJq6\n0lOxvar8fBC4lGpM5IFyWYvy88HSfCWwW8vh44FVQ8Tbfd65tifbntzX19fkV4mIiBYjXlQk7Sjp\nZf3bwFTgdmAe0D+DawZwWdmeBxxdZoFNAR4rl8cWAFMljSkD9FNLLCIiuqQbl792BS6V1P/5/9f2\nTyQtBi6SdCzwW+Dw0n4+cBCwHHgKOAbA9lpJpwGLS7tTba8dua8REREDjXhRsb0CeEub+MPA+9rE\nDRw3yLlmA7ObzjEiIjbN5jSlOCIielyKSkRENKZbU4ojIga177/u2+0UXvSu/eS1HTlveioREdGY\nFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYmIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRE\nNCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTM8XFUnTJN0tabmkk7qd\nT0TElqyni4qkUcA3gAOBPYEjJe3Z3awiIrZcPV1UgL2B5bZX2H4amAtM73JOERFbrF4vKuOA+1v2\nV5ZYRER0wdbdTuAFUpuYn9dImgnMLLtPSrq7o1l1187AQ91Ooi7904xup7A56anfHQCz2v0JbrF6\n6venT2307+5P6jTq9aKyEtitZX88sGpgI9vnAueOVFLdJGmJ7cndziM2Xn53vS2/v0qvX/5aDEyU\ntIekbYEjgHldzikiYovV0z0V289IOh5YAIwCZtte1uW0IiK2WD1dVABszwfmdzuPzcgWcZnvRSq/\nu96W3x8g+3nj2hEREZuk18dUIiJiM5Ki8iIiaZGkyWV7vqTR3c4pNiTpyW7nEBtH0gRJt7eJnypp\n/2GO/aKkz3Yuu81Pz4+pRHu2D+p2DhEvZrZP6XYOm6P0VLqs/CvoLkn/R9Ltkr4raX9J10q6R9Le\nknaUNFvSYkk3S5pejt1B0lxJt0q6ENih5bz3Sdp54L+yJH1W0hfL9iJJZ0q6WtKdkt4u6ZLyuaeP\n9H+LLYkqXy2/89skfajEz5Z0cNm+VNLssn1sfiddNUrSNyUtk/TT8rd3nqTDACQdVP6Or5F0lqQf\ntRy7Z/lbWyHpU13Kf8Skp7J5eC1wONVd/4uB/wa8CzgY+AJwB/Bz2x8pl7RukPQz4GPAU7bfLOnN\nwE2b8NlP2363pBOAy4C9gLXAryWdafvhF/rloq1DgUnAW6juxF4s6WrgauDPqO63GgeMLe3fRbW2\nXXTHROBI2x+VdBHwV/1vSNoe+Dfg3bbvlXTBgGNfB7wXeBlwt6RzbP/nSCU+0tJT2Tzca/s2288C\ny4ArXE3Luw2YAEwFTpK0FFgEbA/sDrwb+HcA27cCt27CZ/ffLHobsMz2atvrgBVsuFpBNOtdwAW2\n19t+ALgKeDvwC+DPymrbdwAPSBoLvAP4ZdeyjXttLy3bN1L9XfZ7HbDC9r1lf2BR+bHtdbYfAh4E\ndu1opl2WnsrmYV3L9rMt+89S/Y7WA39le4M1yyRBm7XOBniGDf/xsP0gn936ua2fHZ3RduEl27+T\nNAaYRtVr2Qn4IPCk7SdGML/YUOvfxnpaLjUzyO9yiGNf1H9X6an0hgXAJ1WqiKS3lvjVwFEl9kbg\nzW2OfQDYRdIrJW0H/MUI5BvDuxr4kKRRkvqoep03lPd+BZxY2vwC+Gz5GZunu4BXS5pQ9j/UvVS6\n70VdMV9ETgP+Gbi1FJb7qIrDOcC3Jd0KLOW5/yn9ke3/lHQqcD1wL9UfQHTfpVSXtG6h6m3+je3/\nKO/9Aphqe7mk31D1VlJUNlO2/5+k/wH8RNJDtPk73JLkjvqIiBdI0kttP1n+0fcN4B7bZ3Y7r27I\n5a+IiBfuo2UizTLgFVSzwbZI6alERERj0lOJiIjGpKhERERjUlQiIqIxKSoRhaRh71iXdKKkl3Q4\nj0mS2i4IKuk9A9aVauLzGj9nbLlSVCIK2++s0exEYKOKiqRRG5nKJCCrTEdPSlGJKPqfdVL+5b5I\n0sVl5dnvllWFPwW8CrhS0pWl7VRJv5J0k6TvSXppid8n6RRJ1wCHS3qNpJ9IulHSLyS9rrQ7vKxU\nfEtZLXpb4FSqu+2X9q9ePEi+g61efb2kN7S0WyRpr8HaRzQpd9RHtPdW4A3AKuBaYF/bZ0n6NPBe\n2w9J2hn4O2B/27+X9Hng01RFAeAPtt8FIOkK4OO275G0D3A2sB9wCnBAWfNrtO2nJZ0CTLZ9/DA5\n/i3tV6+eS7Ve2KyyGOWrbN8o6cuDtI9oTIpKRHs32F4JUG5qmwBcM6DNFGBP4NqyLNu2VOt29buw\nHP9S4J3A90o7gO3Kz2uB88py6pdsZI5TgYP13JMF+1evvghYCMyiKi7fG6Z9RGNSVCLaq7OyrICF\nto8c5By/Lz+3Ah61PWlgA9sfLz2X9wNLJT2vzRBEm9WrASQ9XJ6x8yGq5+4M2l7Si3op9hhZGVOJ\n2DhPUD1sCeA6YF9JrwWQ9BJJfzrwANuPA/dKOry0k6S3lO3X2L6+PJr2Iapn2LR+xlAGW70aqktg\nfwO8wvZtNdpHNCJFJWLjnAtcLulK22uAvwYuKCtFX0f1wKZ2jgKOlXQL1fpQ/YPkX1X1OOHbqZa6\nvwW4kuoRtEMO1FOtXr0N1erVt5f9fhcDR1BdCqvTPqIRWfsrIiIak55KREQ0JkUlIiIak6ISERGN\nSVGJiIjGpKhERERjUlQiIqIxKSoREdGYFJWIiGjM/wdiK/SolVgeFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1035139d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Target分布，看各类样本分布是否均衡\n",
    "sns.countplot(train.interest_level)\n",
    "pyplot.xlabel('interest level')\n",
    "pyplot.ylabel('Number of occurrences')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征工程\n",
    "### 1. 将类别性标签interest_level编码为数字\n",
    " 去掉对interest_level没有用的listing_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_map = {'low': 2, 'medium': 1, 'high': 0}\n",
    "train['interest_level'] = train['interest_level'].apply(lambda x:y_map[x])\n",
    "\n",
    "y_train = train['interest_level']\n",
    "train.drop(['listing_id', 'interest_level'], axis = 1, inplace = True)\n",
    "\n",
    "test_id = test['listing_id']\n",
    "test.drop(['listing_id'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. price, bathrooms, bedrooms\n",
    "数值型特征，+/-/*/ / 特征的单调变换对XGBoost不必要"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_noise(df):\n",
    "    df = df[df.price < 10000]\n",
    "    \n",
    "    df.loc[df[\"bathrooms\"] == 112, \"bathrooms\"] = 1.5\n",
    "    df.loc[df[\"bathrooms\"] == 10, \"bathrooms\"] = 1\n",
    "    df.loc[df[\"bathrooms\"] == 20, \"bathrooms\"] = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 构造新特征：房间价格\n",
    "def create_price_room(df):\n",
    "    df['price_bathrooms'] = (df[\"price\"])/(df[\"bathrooms\"] + 1.0)\n",
    "    df['price_bedrooms'] = (df[\"price\"])/(df[\"bedrooms\"] +1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 构造新特征：房间数的差和\n",
    "def create_room_diff_sum(df):\n",
    "    df[\"room_diff\"] = df[\"bathrooms\"] - df[\"bedrooms\"]\n",
    "    df[\"room_num\"] = df[\"bedrooms\"] + df[\"bathrooms\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 创建日期"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_created_date(df):\n",
    "    df['Date'] = pd.to_datetime(df['created'])\n",
    "    df['Year'] = df['Date'].dt.year\n",
    "    df['Month'] = df['Date'].dt.month\n",
    "    df['Day'] = df['Date'].dt.day\n",
    "    df['Wday'] = df['Date'].dt.dayofweek\n",
    "    df['Yday'] = df['Date'].dt.dayofyear\n",
    "    df['hour'] = df['Date'].dt.hour\n",
    "    \n",
    "    df.drop(['Date', 'created'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 简单丢弃，也可以参照feature特征处理方式\n",
    "def procdess_description(df):\n",
    "    df.drop(['description'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. manager_id\n",
    "将manager分级，top 1%，2%，5，10，15，20，25……"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_manager_id(df):\n",
    "    managers_count = df['manager_id'].value_counts()\n",
    "    \n",
    "    df['top_1_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 99)] else 0)\n",
    "    df['top_2_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 98)] else 0)\n",
    "    df['top_5_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 95)] else 0)\n",
    "    df['top_10_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 90)] else 0)\n",
    "    df['top_15_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 85)] else 0)\n",
    "    df['top_20_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 80)] else 0)\n",
    "    df['top_25_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 75)] else 0)\n",
    "    df['top_30_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 70)] else 0)\n",
    "    df['top_50_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[managers_count.values >= np.percentile(managers_count.values, 50)] else 0)\n",
    "    \n",
    "    df.drop(['manager_id'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. building_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_building_id(df):\n",
    "    df.drop(['building_id'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. latitude & longtitude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_location_train(df):   \n",
    "    train_location = df.loc[:,[ 'latitude', 'longitude']]\n",
    "    \n",
    "     # Clustering\n",
    "    kmeans_cluster = KMeans(n_clusters=20)\n",
    "    res = kmeans_cluster.fit(train_location)\n",
    "    res = kmeans_cluster.predict(train_location)\n",
    "\n",
    "    df['cenroid'] = res\n",
    "\n",
    "    # L1 distance\n",
    "    center = [ train_location['latitude'].mean(), train_location['longitude'].mean()]\n",
    "    df['distance'] = abs(df['latitude'] - center[0]) + abs(df['longitude'] - center[1])\n",
    "    \n",
    "    #原始特征也可以考虑保留，此处简单丢弃\n",
    "    df.drop(['latitude', 'longitude'], axis=1, inplace=True)\n",
    "    \n",
    "    return kmeans_cluster,center"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_location_test(df, kmeans_cluster, center):   \n",
    "    test_location = df.loc[:,[ 'latitude', 'longitude']]\n",
    "    \n",
    "     # Clustering\n",
    "    res = kmeans_cluster.predict(test_location)\n",
    "\n",
    "    df['cenroid'] = res\n",
    "\n",
    "    # L1 distance\n",
    "    df['distance'] = abs(df['latitude'] - center[0]) + abs(df['longitude'] - center[1])\n",
    "    df.drop(['latitude', 'longitude'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. display_address"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_display_address_train_test(df_train, y_train, df_test):\n",
    "    n_train_samples = len(df_train.index)    \n",
    "    df_train_test = pd.concat((df_train, df_test), axis=0)\n",
    "\n",
    "    lb = LabelEncoder()\n",
    "    lb.fit(list(df_train_test['display_address'].values))\n",
    "    df_train_test ['display_address'] = lb.transform(list(df_train_test['display_address'].values))\n",
    "    \n",
    "    #import pdb\n",
    "    #pdb.set_trace()\n",
    "    #A Preprocessing Scheme for High-Cardinality Categorical Features,平均数编码：针对高基数定性特征（类别特征）的数据预处理\n",
    "    me = MeanEncoder(['display_address'], target_type='classification')\n",
    "    \n",
    "    #下列函数并没有对test部分进行编码，因为fit还是根据y_train来训练的\n",
    "    #放弃该函数，直接丢弃算了，文本特征编码需要额外的功力\n",
    "    df_train_test = me.fit_transform(df_train_test, y_train)\n",
    "\n",
    "    df_train_test.drop(['display_address'], axis=1,inplace = True)\n",
    "    \n",
    "    df_train = df_train_test.iloc[:n_train_samples, :]\n",
    "    df_test = df_train_test.iloc[n_train_samples:, :]\n",
    "    \n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_display_address(df):\n",
    "    df.drop(['display_address'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. street_address"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 和display_address信息冗余，去掉\n",
    "def procdess_street_address(df):\n",
    "    df = df.drop(['street_address'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10. features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_features_train_test(df_train, df_test):\n",
    "    n_train_samples = len(df_train.index)\n",
    "    \n",
    "    df_train_test = pd.concat((df_train, df_test), axis=0)\n",
    "    df_train_test['features2'] = df_train_test['features']\n",
    "    df_train_test['features2'] = df_train_test['features2'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "    c_vect = CountVectorizer(stop_words='english', max_features=200, ngram_range=(1, 1), decode_error='ignore')\n",
    "    c_vect_sparse = c_vect.fit_transform(df_train_test['features2'])\n",
    "    c_vect_sparse_cols = c_vect.get_feature_names()\n",
    "\n",
    "    df_train.drop(['features'], axis=1, inplace=True)\n",
    "    df_test.drop(['features'], axis=1, inplace=True)\n",
    "    \n",
    "    #hstack作为特征处理的最后一部，先将其他所有特征都转换成数值型特征才能处理,稀疏表示\n",
    "    df_train_sparse = sparse.hstack([df_train, c_vect_sparse[:n_train_samples,:]]).tocsr()\n",
    "    df_test_sparse = sparse.hstack([df_test, c_vect_sparse[n_train_samples:,:]]).tocsr()\n",
    "    \n",
    "    #常规datafrmae\n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray()[:n_train_samples,:],columns = c_vect_sparse_cols, index=df_train.index)\n",
    "    df_train = pd.concat([df_train, tmp], axis=1)\n",
    "    \n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray()[n_train_samples:,:],columns = c_vect_sparse_cols, index=df_test.index)\n",
    "    df_test = pd.concat([df_test, tmp], axis=1)\n",
    "    \n",
    "    #df_test = pd.concat([df_test, tmp[n_train_samples:,:]], axis=1)\n",
    "  \n",
    "    return df_train_sparse,df_test_sparse,df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_features_test(df, c_vect):\n",
    "    df['features2'] = df['features']\n",
    "    df['features2'] = df['features2'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "    c_vect_sparse = c_vect.transform(df['features2'])\n",
    "    c_vect_sparse_cols = c_vect.get_feature_names()\n",
    "\n",
    "    df.drop(['features', 'features2'], axis=1, inplace=True)\n",
    "    \n",
    "    #hstack作为特征处理的最后一部，先将其他所有特征都转换成数值型特征才能处理\n",
    "    df_sparse = sparse.hstack([df, c_vect_sparse]).tocsr()\n",
    "    \n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray(),columns = c_vect_sparse_cols, index=df.index)\n",
    "    df = pd.concat([df, tmp], axis=1)\n",
    "    \n",
    "    return df_sparse, df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 11. photos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def procdess_photos(df):\n",
    "    df.drop(['photos'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对训练样本和测试样本做特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/pandas/core/indexing.py:517: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "remove_noise(train)\n",
    "create_price_room(train)\n",
    "create_room_diff_sum(train)\n",
    "procdess_created_date(train)\n",
    "procdess_description(train)\n",
    "procdess_manager_id(train)\n",
    "procdess_building_id(train)\n",
    "procdess_photos(train)\n",
    "procdess_street_address(train)\n",
    "procdess_display_address(train)\n",
    "\n",
    "remove_noise(test)\n",
    "create_price_room(test)\n",
    "create_room_diff_sum(test)\n",
    "procdess_created_date(test)\n",
    "procdess_description(test)\n",
    "procdess_manager_id(test)\n",
    "procdess_building_id(test)\n",
    "procdess_photos(test)\n",
    "procdess_street_address(test)\n",
    "procdess_display_address(test)\n",
    "\n",
    "X_train_sparse,X_test_sparse,train,test = procdess_features_train_test(train,test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 特征处理结果存为文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.concat([train, y_train], axis=1)\n",
    "test= pd.concat([test_id,test],axis = 1)\n",
    "train.to_csv('RentListingInquries_FE_train.csv', index=False)\n",
    "test.to_csv('RentListingInquries_FE_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "#数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "train = pd.read_csv('./AIData/RentListingInquries_FE_train.csv')\n",
    "test = pd.read_csv('./AIData/RentListingInquries_FE_test.csv')\n",
    "\n",
    "y_train = train['interest_level']\n",
    "X_train = train.drop(['interest_level'], axis = 1)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_train, y_train, train_size=0.05)\n",
    "#print len(X_train), len(y_train), len(X_test)\n",
    "X_test = test\n",
    "\n",
    "ss_X = StandardScaler()\n",
    "X_train = ss_X.fit_transform(X_train)\n",
    "X_test = ss_X.fit_transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型训练\n",
    "\n",
    "### default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is: [ 0.77664435  0.83472007  0.8036208   0.89170057  0.79830431]\n",
      "cv logloss is: 0.820998018123\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()\n",
    "\n",
    "# 交叉验证：缺省为StratifiedKFold\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv = 5, scoring = 'neg_log_loss')\n",
    "\n",
    "print 'logloss of each fold is:', -loss\n",
    "print 'cv logloss is:', -loss.mean() "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正则化的Logistic Regression及参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "penaltys = ['l1', 'l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid = GridSearchCV(lr_penalty, tuned_parameters, cv=5, scoring = 'neg_log_loss')\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Users/hankaei/anaconda3/envs/py27/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([  2.02287674e-02,   8.23478699e-02,   3.19982052e-02,\n",
       "          1.49384212e-01,   1.62741613e-01,   3.13873196e-01,\n",
       "          1.81361761e+00,   7.82105255e-01,   1.00665310e+01,\n",
       "          1.63708777e+00,   3.94910654e+01,   2.69529700e+00,\n",
       "          6.07334871e+01,   3.20169272e+00]),\n",
       " 'mean_score_time': array([ 0.00119166,  0.00116019,  0.00127001,  0.00118999,  0.00121074,\n",
       "         0.00130944,  0.00119758,  0.00125694,  0.00130439,  0.00120778,\n",
       "         0.00125437,  0.00124459,  0.00127535,  0.0013236 ]),\n",
       " 'mean_test_score': array([-1.09861229, -0.95178192, -0.79641071, -0.78894959, -0.7430872 ,\n",
       "        -0.77912339, -0.78776618, -0.8209856 , -0.88312638, -0.88365485,\n",
       "        -0.97426515, -0.950155  , -1.07932885, -1.0185862 ]),\n",
       " 'mean_train_score': array([-1.09861229, -0.92426983, -0.79408915, -0.71662084, -0.68142657,\n",
       "        -0.64872945, -0.63700705, -0.63414604, -0.63102282, -0.63090609,\n",
       "        -0.629644  , -0.62976361, -0.62949954, -0.62945344]),\n",
       " 'param_C': masked_array(data = [0.001 0.001 0.01 0.01 0.1 0.1 1 1 10 10 100 100 1000 1000],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'param_penalty': masked_array(data = ['l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2'],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'rank_test_score': array([14, 10,  5,  4,  1,  2,  3,  6,  7,  8, 11,  9, 13, 12], dtype=int32),\n",
       " 'split0_test_score': array([-1.09861229, -0.94679351, -0.79727806, -0.77184611, -0.73763774,\n",
       "        -0.74748388, -0.76986892, -0.77664435, -0.83748942, -0.82292245,\n",
       "        -0.92785489, -0.87789095, -1.00654692, -0.94104841]),\n",
       " 'split0_train_score': array([-1.09861229, -0.92547569, -0.79382858, -0.71901119, -0.68604401,\n",
       "        -0.65204917, -0.6390577 , -0.63671541, -0.63346713, -0.63329984,\n",
       "        -0.63206297, -0.63218004, -0.63191793, -0.63188261]),\n",
       " 'split1_test_score': array([-1.09861229, -0.95409788, -0.7973251 , -0.7928461 , -0.73513227,\n",
       "        -0.77703853, -0.78212876, -0.83472007, -0.87792542, -0.92491662,\n",
       "        -0.98967964, -1.02285642, -1.0719788 , -1.14326806]),\n",
       " 'split1_train_score': array([-1.09861229, -0.9249403 , -0.79563836, -0.71877643, -0.68653373,\n",
       "        -0.65316393, -0.64083134, -0.63857248, -0.63604295, -0.63605406,\n",
       "        -0.63460699, -0.63479958, -0.63443676, -0.63440082]),\n",
       " 'split2_test_score': array([-1.09861229, -0.93731529, -0.79636845, -0.76219359, -0.72177765,\n",
       "        -0.75369439, -0.77327685, -0.8036208 , -0.91410097, -0.88903554,\n",
       "        -1.04290784, -0.98171728, -1.16291615, -1.05367137]),\n",
       " 'split2_train_score': array([-1.09861229, -0.92782589, -0.79412881, -0.71837278, -0.68370842,\n",
       "        -0.64801925, -0.63502329, -0.63136366, -0.62705563, -0.62701277,\n",
       "        -0.62559678, -0.62570822, -0.62543691, -0.62538836]),\n",
       " 'split3_test_score': array([-1.09861229, -0.96847748, -0.79534305, -0.83289765, -0.78923283,\n",
       "        -0.85309555, -0.84081339, -0.89170057, -0.92127799, -0.93552643,\n",
       "        -0.9760814 , -0.97674891, -1.12910786, -1.01983721]),\n",
       " 'split3_train_score': array([-1.09861229, -0.91992372, -0.78991813, -0.71048023, -0.66574819,\n",
       "        -0.63812858, -0.62806639, -0.62499066, -0.62211975, -0.62197241,\n",
       "        -0.62085434, -0.62095649, -0.62072108, -0.62067806]),\n",
       " 'split4_test_score': array([-1.09861229, -0.95223087, -0.79573529, -0.78499127, -0.73168268,\n",
       "        -0.76437302, -0.7727907 , -0.79830431, -0.8649412 , -0.84591269,\n",
       "        -0.93486486, -0.89156056, -1.02625705, -0.9350103 ]),\n",
       " 'split4_train_score': array([-1.09861229, -0.92318357, -0.79693184, -0.71646357, -0.6850985 ,\n",
       "        -0.65228629, -0.64205651, -0.639088  , -0.63642861, -0.63619137,\n",
       "        -0.63509891, -0.6351737 , -0.63498505, -0.63491736]),\n",
       " 'std_fit_time': array([  2.10358805e-03,   5.11624444e-03,   1.70278294e-03,\n",
       "          3.92149740e-03,   5.50949522e-02,   2.15260872e-02,\n",
       "          8.77538438e-01,   1.94565227e-02,   1.99951418e+00,\n",
       "          6.74700384e-02,   2.53125524e+00,   2.75449161e-01,\n",
       "          1.16122415e+00,   3.22679581e-01]),\n",
       " 'std_score_time': array([  4.40367312e-05,   1.26135822e-05,   1.03661525e-04,\n",
       "          8.13240706e-05,   5.16794255e-05,   2.06857730e-04,\n",
       "          4.69075452e-05,   6.80316459e-05,   8.26060340e-05,\n",
       "          4.23031158e-05,   2.86036325e-05,   1.18414077e-04,\n",
       "          7.98213724e-05,   1.64147034e-04]),\n",
       " 'std_test_score': array([  1.71925279e-16,   1.01795412e-02,   7.98361477e-04,\n",
       "          2.43676464e-02,   2.36832024e-02,   3.83081684e-02,\n",
       "          2.68257304e-02,   3.99231042e-02,   3.11634195e-02,\n",
       "          4.36766232e-02,   4.16097264e-02,   5.59516313e-02,\n",
       "          5.93787616e-02,   7.71941616e-02]),\n",
       " 'std_train_score': array([  1.40433339e-16,   2.63126488e-03,   2.36422623e-03,\n",
       "          3.19879865e-03,   7.89816871e-03,   5.58961768e-03,\n",
       "          5.06355789e-03,   5.33255379e-03,   5.57498484e-03,\n",
       "          5.57017887e-03,   5.54755576e-03,   5.55843147e-03,\n",
       "          5.54764530e-03,   5.54552011e-03])}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.743087196208\n",
      "{'penalty': 'l1', 'C': 0.1}\n"
     ]
    }
   ],
   "source": [
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3XlYVcX/wPH3XFZZFRBRUEBxARRR\nESE1991yyyUzyzJzyTLLtLJNbdPKtCwzyy2X/Fqmue9LCijuIu6IIqiIICI7zO8P0J8lyr1wL5dl\nXs9zHu49d87MBx5lOGdmPiOklCiKoiiKtjTGDkBRFEUpW1THoSiKouhEdRyKoiiKTlTHoSiKouhE\ndRyKoiiKTlTHoSiKoujEaB2HEMJBCLFVCHEu/2uVAsq0E0IcfeBIF0L0Nka8iqIoSh5hrHUcQojp\nwC0p5RdCiElAFSnlxMeUdwDOA25SytSSilNRFEX5N2M+quoFLMp/vQgo7E7iGWCj6jQURVGMy9SI\nbVeTUsYBSCnjhBDOhZQfBHyjTcVOTk7Sw8OjmOEpiqJULIcOHboppaxaWDmDdhxCiG2ASwEfva9j\nPdWBRsDmx5QZAYwAqFWrFuHh4bo0oSiKUuEJIaK1KWfQjkNK2fFRnwkhrgshquffbVQHbjymqgHA\naill1mPamgfMAwgICFAJuBRFUQzEmGMca4EX8l+/AKx5TNlngeUGj0hRFEUplDE7ji+ATkKIc0Cn\n/PcIIQKEEPPvFRJCeAA1gd1GiFFRFEX5D6MNjkspE4AOBZwPB4Y/8P4S4FpykSmKUp5kZWURExND\nenq6sUMpNSwtLXFzc8PMzKxI1xtzVpWiKIrBxcTEYGtri4eHB0IIY4djdFJKEhISiImJwdPTs0h1\nqJQjiqKUa+np6Tg6OqpOI58QAkdHx2LdgamOQ1GUck91Gv9W3J+H6jgURVH+Y+BPIQz8KcTYYZRa\nquNQSj31n1gp62xsbO6/7tq1K5UrV6Znz573z/Xp0wd/f3+8vLywt7fH398ff39/9u/fr1M7O3bs\nIDQ0VG9xP4oaHFcURSlBEyZMIDU1lZ9++un+udWrVwOwa9cuvvrqK9atW1ekunfs2IGTkxNBQUF6\nifVR1B2HoihKCerQoQO2trZalz948CBt2rShWbNmdOvWjevXrwMwc+ZMfHx8aNy4MUOGDOHChQvM\nnz+fGTNmFOluRRfqjkNRlArjk78jOBWbXGi5U3F5ZbR5ROpTw46PnvItdmwFycjI4I033mDt2rU4\nOTmxdOlSPvjgA+bNm8f06dOJjo7G3NycpKQkKleuzPDhw3FycmLcuHEGiece1XEoiqKUUpGRkURE\nRNCxY17av5ycHNzc3ADw9fVlyJAh9OrVi969S3Z/O9VxKIpSYWh7Z3DvTuP3V4MNGU6hpJT4+fmx\nd+/ehz7bvHkzu3fvZs2aNUybNo2TJ0+WWFxqjENRFKWU8vHx4erVqxw4cACAzMxMIiIiyMnJISYm\nhvbt2zNjxgzi4+NJTU3F1taWO3fuGDwu1XEoiqKUoNatW9O/f3+2b9+Om5sbmzc/cpshLCwsWLVq\nFePHj6dx48Y0adKEsLAwsrOzGTx4MH5+fjRt2pSJEydia2tLr169WLlyJU2aNFGD40rF9mHChPxX\n/xg1DkUpqpSUlPuvC3rsdE/btm1p27btv841bdqUf/55+N/+vn37HjrXoEEDTpw4UfRAtaQ6DkVR\nlP8w9thGaaceVSml3kybXOZbmiBzc40diqIoqDsOpRS7ejGSq2un8GPiacxEDpenNSTWazDe3UZi\nX8XJ2OEpSoWlOg6l1Ik5f5K4ddNokrgZR0z408SXS+Zm9M6+S9DZGaSemc0Bx844th1NHb8njB2u\nolQ46lGVUmpcOXeMgzMH4LKkNY0StxFerT8prx7iD5c7HHK4Rf3JYZzvs56TDh1plLCZOn924/Sn\nwYSvnUtGeqqxw1eUCkPdcShGF33mKPHrp9Lk9nacMCPcZSBefd4jyKXWQ2W9GreCxq24fSueY5vm\n4npuGQ0OT+TW4U85XKM3Hl1eo7p7fSN8F0q5sqBH3tdh640bRyml7jgUo4k+fZjwr/tSc1lbfG7v\n5WD1waSOPkLQqLk4PdBpRJvVIdqszr+utXeoStDgD3CdfJIT7RcSbdWIwKtLcP61BUend+X4zlXk\n5uSU9LekKAUqibTqc+bMYenSpXqN+1HUHYdS4i5FhpOwYRpNkndRFXPCajxP/T6TCHJ21bkujYkJ\njZ7sA0/24drlc0RtnkO9q3/iuPtlYva8T0ydZ/HuNgp7x2oG+E4URXfFSauenZ2NqWnBv7bHjBmj\n/2AfQd1xKCUmKiKMw189Ta0VHWmQHEKY61AyXjtG8Kvf4VCETuO/XGrVJfiVb7F99yzhATO4Y+pI\n0PmZWMz25eC3gzh3ZI8evgtFKR5d06q7ubkxdepUWrZsyerVq5k7dy7NmzencePG9O/fn7S0NAAm\nT57Mt99+C0CrVq2YNGkSgYGB1K9fX++ryNUdh2JwF06EcnvTNJre3UtVWYkwtxfx7jOJYCcXg7Rn\nbmFJQM8R0HMEF0+GEb/zBxrd3IjVmo2cXV+PJN+h+HUZhqWVTeGVKeXLxklwTYuV1deO5329N9bx\nOC6NoNsXxYurENbW1vdXiickJDBy5EgAJk2axMKFCxk1atRD10gpOXDgAGvXrmXKlCls2rRJb/Go\njkMxmPPH9nFn86c0Sd1HMlaE1ByOT59JBDtULbEYajdsQe2GLUhOSiBs0zxczi4l8Nhkko59wVGX\nXtTsPAbX2obZS0FR9GXgwIH3Xx8/fpwPP/yQpKQk7ty586+xkgf17dsXgGbNmnHp0iW9xqM6DkXv\nzh3dS8qWz2iSup9krAmpNQKfPhMJNuKiPbvKjrQY9C4ydyInQ9aTGTKPgLjlaBYt41ilAGTz4TRq\n8wwmj3h+rC+lJV13haXtnUEpm1VlbW19//XQoUPZuHEjDRs2ZP78+Y/cY9zCwgIAExMTsrOz9RqP\n6jgUvTl7eDdpWz+lcVoYt7EmxH0kvn3eIbiyo7FDu09oNDRs+RS0fIobV6O4sGkOXldWUXXvq8Tt\nncwlz4HU7zZaL2MuimIId+/excXFhaysLJYtW0bt2rVLPAbVcZRTwzYNA2BB1wUGb+tM+A7St31G\n4/SDJGFDiMcoGvZ5h2B7B4O3XRzOrp44v/wVWZmfcnj7MiyOLCA46nsy58wl3L4dNq1HUr9Ze4RG\nzSFR9Kd169acPn2alJQU3Nzc+OWXX+jSpYvW10+ZMoXAwEBq1apFw4YNSU9PN2C0BRNSyhJv1NAC\nAgJkeHi4scMwqpLoOE4f3Ebm9s/xSw8nEVtOe75Aoz5vY2NXRa/tlGQnGB15iGvb5+AbvwEbkcZ5\nkzrc8nmehl1ewsrGvtj1q0dVJS8yMhJvb2/dLiplj6oMoaCfixDikJQyoLBr1R2HorPIsM1k7/ic\nRhlHSMSO0Nqv06jPWwTbVjZ2aMXm7t0Md+9fSUlOJGzTfJxP/0bgiY9JPjGdUOencO00hpp1Gxs7\nTMXQynGHoQ+q41C0dipkI7m7vqBhxlESsCfUaxx+vccTpIe/xEsbG7sqtBgwAZn7FpEHt5L6z1ya\nXl+F+dLfOWHRhKymL+PXfiCmZubGDlVRSpzqOJRCRexbD7u/wDfzODepTGjd8TTuPZ4ga+0XMRXH\noO8i8l50LZHm/kVoNHi36AItunDz2mXObZxD7ej/US3kNa6HfMRF9/7U7TbmXylSFKW8Ux2HUiCZ\nm0vE/nVo9kzHN/ME8VQhtN4EGvd6o0Q6jIy0FG5ePc+tq1HEm2cgEOTkZGNiYrx/sk4utXAa9iXZ\nWVM5snMlpod+ITh6Lpk//swhuzZUavkq3oGd1WC6Uu6pjkP5F5mby8l/1mC6dwYNsyK4gQOh9Sfi\n3+t1goq50jojLYWE2Ivcir3InWtXSL0RR2b8dXISbiFu3cY0KQXL5HRskrOxysibtGEKNMu/PrSl\nP7c7BtD4xXG4evkX7xstBlMzc5p0HgKdh3D57FFit/2Az42/sdu0g6gt7txoMATfrq/ofZKAopQW\nquNQgLwO48Se1Zj/M4NG2ZFcx5Ew73dp/PRYgipZP/K6zLRUbsZdIDEuijtxV0i9EUvGjevk3Mrv\nDBLvYJmcjvWdbKzT8zoDE6By/gGQaiG4a2tKup0lKTUdueNgj4mDA+bO1ajkXJ3o5QvIFjlUzbHG\nfVUYSaue5bh3FWz69qbFM69hXsnK4D+fR6lVz59a9eaRmnKbA5t/xfHUYlqc+pSUiG8Iq9qNah3G\n4OH9/5NULpl/lf/qD+MErGilJGfylUWq46jgZG4ux3f/geW+Gfhln+EaTuyrNxG3Zp2xTbjG0Q0L\nSb0RR0b8dXJuJiASb2OamIJFcjrWd7KwScvrDDSAff4BkGYOKbZmpNtZ5HUGVezQODpgUTWvM7Ct\n5kYV19o4VPfEyubxs7HuLPoNMKPL+hCunD3EiUWzsN92GIdPF3B05kJutPGhwdAxeDVpZ8gf1WNZ\n2dgT2O9NZJ83OH14Jyl7fqRJ/FrMf/+TCHM/0v1fwq/jYNyzLhgtRsV4bGxsSElJ4ejRo4waNYrk\n5GRMTEx4//33GThwIH369CEqKoqUlBTi4+Px9PQE4IcffuCJJ7Tf5XLHjh1YWVkRFBRkqG8F0KHj\nEEKcAP676OM2EA5Mk1Im6DMwRb+yMtNJiLtIYmwUydeucPdaDDfPHMMkLgqLtExM0ky4mlYTm7u5\nOKQtIZUlCArqDPLuDFJrVCHFwY54RwfMq1ajkrMLti41qVLDE8cadQrtDHSxYmxeLqkuQM16zaj5\n6WKyP87k0LpfublyBe6bI8jaOJqtHtaYPN2FwOfexMbeOOlNhEZDg4AOENCBWzeucmbTXDwuLqf6\ngXHEH/iELiYe7HFQuxVWVFZWVixevJi6desSGxtLs2bN6NKli9Zp1QuzY8cOnJycSk/HAWwEcoBl\n+e8H5X9NBhYCT+kvLKW4esw9hmlKOju/8Mc6ORPrVHk/h75d/lEdSDeDOzZmpNtXIt3NmlQHe246\nOGDuVJVK1apjU80trzNwrYO1belZCW5qZk6LPiOhz0jir57n8OJvsNi4j2qz/+Ts3D+JDapNrcHD\naPhkXzRGGqx2cHYleOhUcrI/4ujuVYiD8xmWFk63G5WJjjyEu3ezwitRypV69erdf12jRg2cnZ2J\nj4+ncuVH/6F18OBB3n77bVJSUnB2dmbhwoVUq1aNmTNn8vPPP2NmZkajRo345JNPmD9/PiYmJixc\nuFDnuxVd6NJxtJRStnzg/QkhxD4pZUshxBB9B6YUXXZWJja30qmaJLnubc9dbzs0DlVIyU7F6u4Z\naprdJMvCljS/YQQ9Mx4zcwtjh1wsVV296PLuD+ROzOXEzlXELluAW8hFzPZ8wJ5qU8no/iQBQ8fj\nWN3TKPGZmJri32EQdBjEGzOf5P2kKGx+f4pTXX7BJ7ibUWKqqL488CWnb50utNy9MvfGOh6ngUMD\nJgZO1DmWAwcOkJmZSZ06dR5ZJiMjgzfeeIO1a9fi5OTE0qVL+eCDD5g3bx7Tp08nOjoac3NzkpKS\nqFy5MsOHD8fJyYlx48bpHI8udOk4bIQQLaSUYQBCiEDg3jQb/aZeVIpl+1dv4n5DEu5txnPLdnBs\n22/YHZhJnZwoYqxciPV7jyY9RpT5DuO/NBoNjTsMoHGHASTfusbBpd+S+/dWai3YxtXF2wj1r07V\nAYNo1vMlo03rjbNO5n0TNz5Iuo3XpiEcSvyKZt0L/+WklC9xcXE8//zzLFq06LF3xJGRkURERNCx\nY0cAcnJycHNzA8DX15chQ4bQq1cvevfuXSJx36PL/57hwK9CCBtAkPeIargQwhr4XNeGhRAOwO+A\nB3AJGCClTCyg3HSgB3njr1uBN2R5TLClJxdP/IPL0h0c9RJEP+FI9KdNaZJ7iSuiBgebfE6T7sNx\nqwCrne0cXOgw9gsY+wVnw7dxYcmPVNsTic3EmYR+PpvbnQLwf3E8Ner4lWhc0WZ1wAzsBq/m4k99\naBL2JqFJMQQN/qBE46iotL0zMOSsquTkZHr06MG0adMKHYuQUuLn58fevXsf+mzz5s3s3r2bNWvW\nMG3aNE6ePKn3WB9F64e/UsqDUspGgD/gL6X0k1IekFLelVKuLELbk4DtUsq6wPb89/8ihHgCaAn4\nAQ2B5kCbIrRVIeTkZHP+nTfJMhWcC3Bg8t3DmJBFeNMvqfH+CZr3Gl0hU2TUC+hIz1l/4Lf/APHv\nDeVONVs8/xdGYo+BbOz7BHuXTCczrWQHrCs7ueDx5laO2bQk6OxXhP74Krk5OSUag1LyMjMz6dOn\nD0OHDqV///6Flvfx8eHq1ascOHDg/vURERHk5OQQExND+/btmTFjBvHx8aSmpmJra8udO3cM/W1o\n33EIISyEEIOBMcDrQogPhRAfFqPtXsCi/NeLgILutSRgCZgDFoAZcL0YbZZr2795i5pRKdwZ9Qwd\nSOOqdMD1veMEPD3S4BsUlQUWlWx4cui7dF0Tgs3a34ju1xz7mGScPl3AkVYBrH+zP+eP7CqxeCyt\nbPB7cw1hVZ8h6PoKjszsS3ra3RJrXyl5K1euZM+ePSxcuBB/f3/8/f05evToI8tbWFiwatUqxo8f\nT+PGjWnSpAlhYWFkZ2czePBg/Pz8aNq0KRMnTsTW1pZevXqxcuVKmjRpovd9xh+kdVp1IcQm8qbf\nHiJvdhUAUsqvi9SwEElSysoPvE+UUj601FYI8RV5j8kE8L2U8v1H1DcCGAFQq1atZtHR0UUJq8yK\nPhVG4sAXiavrQMCc5VSZ25gV1sEMeWeDsUMrNkM+NsjO+v9pvbWOXcc0F654WGNqoGm9LRb0AyBs\n2P8vAJS5uYQt/ZigC7M4Zd4I15GrsS/B7XXLu6KkVa8ICwBLKq26m5RSpzRzQohtgEsBHxX4y7+A\n670Ab8At/9RWIcSTUso9/y0rpZwHzIO8/Th0ibOsy8nJ5vQ7r+OkgcZfzeXCrt8IErkcruGEmu72\neAVN67Xc8A/OD0zrdR/8Er5P9jHYtF6h0RD0/BTC/3bFL/xdYr9vR9qLq3GpVdcg7SmFK88dhj7o\n8j9hvxCikS6VSyk7SikbFnCsAa4LIaoD5H+9UUAVfYBQKWWKlDKFvLUkhl3ZUgbtmD2JWueTSXql\nNzVqN6LKhb84gyuJTibGDq1MuTett/Xuo2TO+YTY5h64hVzEdORkdrdvyuYvx5IQF2Ww9gOeepWz\nnRfhmHMTza+duXgyzGBtKUpx6NJxtAIOCSHOCCGOCyFOCCGOF6PttcAL+a9fANYUUOYy0EYIYSqE\nMCNvYDyyGG2WO1fOHsLp1/VE169Mu9GfEnP+JPWzz7DLVv21WlT3pvX2/GUjtXfvJHZML3LMTfKm\n9XbszvohHTiw9mdycvQ/C71hy6e4OSDvv0LV//Xi5D9r9d6GohSXLh1HN6Au0Jm8VeI9Kd5q8S+A\nTkKIc0Cn/PcIIQKEEPPzy6wCLgAngGPAMSnl38Vos1zJzc0lYsJrSAGNvv4RjUbDlT2LyZWC4zXL\n/m58pcG9ab0dtxxCs2QWlzt44xwRh+073xDSyp8NH7xI7IXi/P30ME/fFuS+vJUEk6rU2/oi4X//\npNf6FaW4tB7jkFJGCyGaknfnIYF9UsrDRW04P7dVhwLOh5M3GI6UMgd4tahtlHc757yH+5kkro7s\nSVMvf2RuLm5X/ibSohGpdmpPCH2r37wz9Zt3Ji01mYMr55C6em3etN7/DeSYjwO2fXsT2G+MXrL1\nutT04vbYnZz7sTcBh94hJDGGoCGfqL0+lFJBl+m4H5I3bdYRcAIWCCEmGyow5fFiLxynys9ruFzX\njnZj89Zfnju6l5oylrv1+xo5uvKtkpUdT76YN63X+q/FRPcNwD4mCcdpv+p1Wq99FSe8xm/hkG07\ngi/O5sAPL5OTrZI0lITo54cS/fxQY4dRauny58uzQHMp5UdSyo/IG6R+zjBhKY+Tm5vLsQkj0Uho\nMP27++kzboX+RqY0pX7751nQdYGaGVICajVoTvfPlhC47wi3PxtLQl1nam0+Sdazo9jSrTk7fpxM\nyu3/TxztU90On+p2WtdvYWlFk3F/EFrtWVrc/JPjM3uRnppiiG9FMSAbm7zsTEePHiU4OBhfX1/8\n/Pz4/fffAejTpw/+/v54eXlhb29/f42HLmsx5syZw9KlSw0S/3/pMh33EnmL8dLz31uQN/6glLBd\n8z7C41QiV4Z3wd87EMhbj+B1YwsRNkE0qWKclOKGUhY6QFMzc4L6joa+o7kRc5bDi2ZSadM+nGf9\nwdkf/8ib1vvccAbOPoEQGp32T9eYmBA0ai6hy9wIPPMVZ7/piMvIv6jsVNBMd6U0K25a9ezsbEwf\nsZh3zJgxBov7v3S548gAIoQQC4UQC4CTQIoQYrYQYrZhwlP+Ky7qJPY//sGV2rZ0ePOr++cj96/D\niSRko8LTGCiG5exWj67v/0jr3UfJmPMRsQHuedN6X30Ps6S73DBPL9KMrKDBkzkaNBPPrPPcmdOO\n2KjCs7wqpUu9evWoWzdvxuODadUfx83NjalTp9KyZUtWr17N3Llzad68OY0bN6Z///6kpaUBMHny\nZL799lsAWrVqxaRJkwgMDKR+/fp6X0Wuyx3H6vzjnl16jUQpVG5uLkcmjKJGjqTejFn/yvCafngF\nyVjh00Z1HKWFRqPBv8Mg/DsMIvnWNQ78NhO5ci0BkVns7NaCau+9R6O2/XSqs2m3YZyqUh3XTS+R\ntagL5/ssxatxKwN9B+XPtc8+IyOy8A43/XReGW3GOSy8G+Dy3ns6x6JNWvV7rK2t2bdvHwAJCQmM\nHDkSgEmTJrFw4UJGjRr10DVSSg4cOMDatWuZMmUKmzZt0jnGR9ElyeGixx16i0h5pN2/TMXz5E2u\nD+mAh2/w/fNpd+/gk7Sb01XaYfmY/cEV47FzcKHj61+SY2dNuLcZ1onpmI6czLphXbgerdvSJJ+g\nriQNWkc2ZlT/sx/Hd6n9y8uae2nVFyxYoFVGgoEDB95/ffz4cVq3bk2jRo1YsWIFERERBV7Tt2/e\nJJlmzZpx6dIlvcR9T7Ey3wkhPpZSfqynWJTHuHHlDLZzfifGw5oOb33zr88idq0gQKRjFfCskaJT\ntCWEBudMSxpt28DuT1+n1vojXH2qL0f6t6bd299gUcmm8EoA9wZNiR+xjWu/9MZn53AOJMYS2Ges\ngaMv+7S9M7h3p+G+ZLHeY9Alrfo91tb//wfh0KFD2bhxIw0bNmT+/PmEhoYWeI2FRd5+OyYmJmTr\neTZecSeFH9JLFMpj5ebmEj5hBGZZEs8vv34oNbrpyVXcwAGfoO5GilDR1oqxvqwY64uNvRM9pi/D\n/n8LudbACfelewnrGMz+lbPJzc3Vqq6qNTxwfn0HkZaNCTw2mZAFE5FaXqsYh65p1Qty9+5dXFxc\nyMrKYtmyZYVfYADF6jjUKu6S8c/iL/E8eoO4wW3wavzv7UgS4+PwTT3IRZduaExUbqqyxt2nBd1X\n7uXOl2+Sa6Khyoc/srlfK84f263V9bb2DtQfv4mD9p0Jjp7Lwe+Hkp2VaeColaLSNa16QaZMmUJg\nYCCdOnXCx8fHQJE+XqFp1fN34LsopZz7n/NvAi5SSt032zWwgIAAGR4ebuww9CL+6nku9Xya21Ur\n0XZ9yEN3G2Erp9Pi1Kdc6LuROn6G2Zhe0Z/HpevOzEhl57cTcVq2HfMsSXQXX1pP/g57x+qF1itz\ncwmdP47g2EUcq9SCumP+h5WNvd7jL4uKklbdkI+qSovipFXX5o6jJ/npyv9jFnlbuioGdGDCK1hk\nSjy+/KrA3fvszq3mkqYmtRuqpMFlnbmFFV0mfketDWuJblUbz40RnO7cge1z3it0+q7QaAgeMZsw\nn/dpmHqAmG87knA9poQiL3/clywu151GcWnTcUgp5UMPTvPPCf2HpNzzz28zqH34Glf7t8SrSbuH\nPo+9dAbvrFPEuT+tchiVI1Vdveg5bz3i1xncdramxner2dUlkGPbC9+hucWAdzje8ntqZl0ibW4H\nYs6X3D7USsWhzW+bVCHEQzm688+l6T8kBSAhLgqzmQuIdbWkw7sFr6+8vCtvFrRH2xcK/Fwp23ye\n6EnH9WFcnzCYSskZmI/5iHUvdCIu6vGdQZPOQ4juuRxreRfr37px9vCukgm4FNN2p9OKorg/D206\njg+BjUKIF4UQjfKPYcD6/M8UAwiZ+ApWaRLXz7/A3OLhbKsyN5fql9cSaeZLdff6RohQKQkajYa2\nL3+A39Y9XOzdlJrhMVx7uj8bPx5OWmryI69r0LwjKc+tJ01Uouaa/hzdvkLvsQ3bNOz+mE1pZmlp\nSUJCguo88kkpSUhIwNLSssh1FLqOQ0q5UQjRG5gA3JsofhLoJ6U8UeSWlUfat2ImdQ5cJap/C7oH\ndimwzIUTIXjlXiGsgUpQXBHY2DvS44ulRL9wgIhPJuC5Yh/hW1qiGTuM4IHjClxEVrNuY26+up2Y\nn3vTaM9Iwm5dpUX/t4wQvXG5ubkRExNTaGqPisTS0hI3N7fCCz6CVgsApZQn+f/d+gokhPhOSqlW\nIBVT4o3LmHw1n7jqlnSc/MMjy93cvwR3aUL99s+XYHSKsbl7B+K+YjcH1/1KzoxZOH3yM1tW/A+v\njz8rcBzMyaUmlcZt5+ScZ2gRMYWQ2zEEvfR1hRoTMzMzw9PT09hhlCv6/NfTUo91VVj7Jr6CdWou\n1T+bVuAjKoCc7GxqX99MhHWgypBaQTXv+RItt4ZxZXgXnKJvkz54NOte70vSzasPlbW2rYzP+PUc\nqNyd4JhfCZ/1LFmZGUaIWikvKs6fHWVA6Ko51Am5zOVezfAOfvRM58jQDThzixzfZ0owOqW0MTO3\npPPb3+K5cT2X2nrhuSWSs507sW32xIcWAZqZW9D89aWE1BpB89ubiPymGynJiUaKXCnrVMdRStxO\niCN3+g9cr2ZOxw/nPrZsavhyUmQlfNoOfGw5pWJwrO5Jzx//RrPwG5JcbHD9YS27u7bg6Nbl/yon\nNBqCX5rBgUaf4JN2hGuzOnCQRCv/AAAgAElEQVQzNtpIUStlmT47DrWmoxj2TnoZ+zu5OE375LGJ\n7tLT7uKduJPIym2oZG1bghEqpZ13UDc6rgslftJQLFOysBg7hfXPdyD2wvF/lQvsN46INj9RIzuG\nrHkdiD6jW8qLe07FJXMq7tEzu5Tyq9COQwjxjBBCm3lbs/QQT4V0YM086uyN4lJPfxq27v3Ysqd2\nrcRWpGHZbFAJRaeUJRqNhidffBf/bXuI6tcctyOx3Og9kI0fvkRqStL9co3bD+Bq71WYk4n98h6c\nDttixKiVskabO47ngMtCiMVCiG5CiAIz6UkpF+o1sgoi+dY1sj6bzXVnc9p/8lOh5cWJ/3GTyvg8\n8VQJRKeUVda2DnT/dDEOf/7G1UYueKwM4XCnVvyz9Kv72XfrNnmS9KEbuSPs8NgwmMOblxg5aqWs\nKLTjkFL2AbyA7cDrwBUhxI9CiCcNHVxFsOe9V6icnIPDlA+oZGX32LK3b8XjezeM885dMHnEvsOK\n8qCa9ZrRY9lOUr+ZRJaFKY5Tf2FLryc4czDvDsO1ti/Wo7YRbVYb//1jCVvxuZEjVsoCrcY4pJTJ\n+Tv9dQMaAUeB74QQVwwaXTkXvn4BdXad51LXhvi1LXyG1JkdSzAX2TgGDymB6BRDWNB1QYGZcQ2t\nWfcXaLUllJgR3XGMSSZ76Buse603iTcu4+DsSq03t3PMOpgWp78g5Kcx5ObklHiMStmh0+C4EKIK\n0BcYCDgAas/KIkq5fZP0aV8T72hGu6k/a3WN9dk/uSJqqD2mlSIxM7ek0/ivqbNxI1Ht6uK5/Qzn\nu3Rl68y3MTO3oNGbawhz7E1w3G8cnjWAjPRUY4eslFLaDI7bCiGeF0JsACKB5sA0oJaUcpyhAyyv\ndr0/nCqJOdh9/C5WNpULLX/t8jl8M08QU0tlwlWKx8HFnZ4/rMVk0bfccrXF7af17OkcyLHtvxM4\nZgEhnmMISN7G+W+6kpyUYOxwlVJIm99AUUBX4EegppRyhJRyh1QZw4rs0KYl1Nl2hqjO3vh30m6f\n8KhdeXsD1Goz1JChKRVIg8AudFobws33h2GRmo3VuM/YMKQDNZ/oz8Emn1Mv4yQJs9tzPeaCsUNV\nShltOo6mwGdSyr+llFn3TgohfIUQVQ0XWvl0984tUqdM56aDKW21fEQF4HJpLWdMG+Ba29eA0SkV\njUajofXz79Bk+z9E9W+B6/FrJPQZTPzBAxxrOQvnnGswvxNRpw4aO1SlFNGm45gOOBVw3g21dkNn\nOye/gtOtbKw/mICNvaNW10RFhOGZe4kkr8ev8VCUorKyqUz3qQup+tdyYvxr4LkqjIwPvmSH/SCE\nzMFx5dNE7Ftv7DCVUkKbjqORlHL3f09KKTcDfvoPqfw6un0FnptPcaF9XZp20/6R07V/fiNbavBq\npzLhKobl6uVPj9+2k/rte2RYmeE1bwuHD1Tl+N0q1N0ylEPr5xs7RKUU0KbjMCviZ8oDUlOSSP7o\nM5Iqm9D281+1vi43JwfPuA1EVGqGY7Wi589XFF006/o8T24J4+rInjheS6XKOsnfp6rjuf8dQpd+\nYuzwFCPTpuM4J4To/t+TQohuwEX9h1Q+7fzoVarezMJ88nhs7At68lew0we24MJNsnxUJlylZJma\nmdNx3Ay8Nm8iqmMD6p/M4cL6aiRvX8C+74fzwdJIPloaaewwFSPQZvnxm8A6IcQA4FD+uQAgGOhp\nqMDKkxO7/sBjw3EutKlDz54v6XTtnYPLSJUW+LTXbvaVouhbFeda9PxuNWfDtxEzZTLu++Ha2X9I\nb2SJxQW1ULAi0iblyFnyVovvBjzyj92AX/5nymOkpSaT8OEUkuxMePJz3Z4PZ6Sn4n1rO6fsW2Nl\nY2+gCBVFO/UCOtL5r/0kfPAyplkafHdmkGOew5nDDw2BKuWctilHMqSUC6SUb+Ufv0op0w0dXHmw\n4+ORVLuRidm7Y7Fz0G23vlN7/sSOu5g1UZlwldJBo9HQ6rm3CdgZyoGG5nhGSVJeHMlfH7/40OZR\nSvml9RJkIURfIcQ5IcRtIUSyEOKOEEIl43+MiH1rcf/7CBdaehDY+1Wdr5fHf+cWdvi26mWA6BSl\n6CpZ2VE1KYujfjbccIH6K8LY0aU5p/avM3ZoSgnQJXfFdOBpKaW9lNJOSmkrpXx8OtcKLDMtlRvv\nf8gdGw2tvtR9CmNyUgK+d0I459QJUzNzA0SoKMXzyXPe/NC5Fo3n/cX5dibYJWUiX57Autf7knzr\nmrHDUwxIl47jupRSTaHQ0rZpI3G5loGYOJrKTq46X39651IsRBb2Qc8ZIDpF0Z8aHvVp9+VecvvU\nIMo3G88tkUR06cDeJdPv7/2hlC/aJDnsK4ToC4QLIX4XQjx771z++SIRQjgIIbbmP/7amp95t6By\nXwohTuYfZWKT7cjQjdRcfZALQTUJemZMkeqwOv0HMcKF+k3b6Tk6RdE/G7sqtHh3M04d2pPbPYm7\nlSROny5g8zOtiY48YOzwFD3T5o7jqfzDDkgFOj9wrjjTcScB26WUdcnbJGrSfwsIIXqQlyvLH2gB\nTBBClOrHY5kZqcS99x53rTS0nF60VbbxsZfwST/GFbeeKhOuUmaYmJoSNOonUoPG07JNHKeDNVQ7\nf4uk/i+wccorZKaVzjTtwzYNY9imYcYOo0wpdB2HlNJQP9FeQNv814uAXcDE/5TxAXZLKbOBbCHE\nMfIy9a40UEzFtu3TMXjGpnPro1eo4lyrSHVc2LGQqkLi9qTKhKuUPS0GTOCEixcddozmqosFZ89V\np96yf9i/LRi79yfQtIvaiKys02VW1SIhROUH3lcRQmifO+Nh1aSUcQD5X50LKHMM6CaEsBJCOAHt\ngJqPiG+EECJcCBEeHx9fjLCK7szBLdRcFcrF5jVo+ez4ItfjFLWWc6Z1qVm3sR6jU5SS0+jJPiQO\nXIe9pTldfc8R+UIwplk5VHrjU9a91JWEuChjh6gUgy7PQfyklEn33kgpE4Emj7tACLHtgfGJBw+t\n5pdKKbcAG4D9wHIgBMh+RNl5UsoAKWVA1aoln+09KzOdmHcnkVpJEFTER1QA0acP45VzgYTaagqu\nUra5ezfDctROLprXpW/GH2QP68TFp/1xD43mYvce7PjhfXJyCvzvrJRyunQcmgcHsIUQDhTyqEtK\n2VFK2bCAYw1wXQhRPb+u6sCNR9TxqZTSX0rZCRDAOR1iLjHbvhhLjZg0Mse9iGN1zyLXE7t3MTlS\n4NX+BT1GpyjG4eDsSu23thNu14m2sb9SzSULk1+/IdHFmuqz/2R7z2DOhm8zdpiKjnTpOL4G9gsh\npgohppB3FzC9GG2vBe79dnwBWPPfAkIIEyGEY/5rP/LSuG8pRpsGcf7ITlx//4eLTV1o/fw7Ra5H\n5ubiHruBU5ZNcHIp2viIopQ2FpZWNBu3khD3kQQkb0Vsn0KTX1cR90Y/qly7S8bQsayfMIiU22qb\n2rJC645DSrkY6AdcB+KBvlLKJcVo+wugkxDiHNAp/z1CiAAhxL1nPWbAXiHEKWAeMCR/oLzUyM7K\nJGri22SYCwJnaL+jX0HOhG+nhrxOurfKhKuUL0KjIXjYlxwK/AbPzHNkzutEnXZ9qb1hPdHBHtT+\n+xjHurQhdNUcY4eqaEGrjkMIoRFCnJRSnpJSfi+l/E5Keao4DUspE6SUHaSUdfO/3so/Hy6lHJ7/\nOl1K6ZN/BEkpjxanTUPYNv0N3C6nkjp2MFVdvYpV1+0Dy0iT5ni3H6yn6BSldGnW/WWin1qJpUyn\nyooexJ47Ss9fNpI2632yzE2wn/w9Gwa1JfbiCWOHqjyGtkkOc4FjQgj1/OQBF0/8Q/Xlu4hq7Ezr\nF98rVl1ZmRnUu7mVU3YtsbErcC2kopQL9QPakzlsKwmaqnhvf4mwlTNo2mUIT2wO4dLgVtQ4eZ1r\nvQew+YvXyMpUuVRLI13GOKoDEUKI7UKItfcOQwVW2mVnZXL+nTfJMhUEfDUPTTEX6kXsXU0V7mDS\nuEwsjlcUfKrb4VO9aOtxq7vXx+mNXURUakaLU9MI/WEEJmbmdPvwZxxWLea6lwO1Fm5nb5dgTuz6\nQ8+RK8Wly2+7T8hbKT6FvIHye0eFtGPm29SMSiFlzACca9Yvdn05R38nCRt8WvfRQ3SKUvrZ2jvg\n+9YGQqv2J+jG75z4ugcpyYnUatCcLqv2kjD5JSqlZKIZOZl1o58m6eZVY4es5NNlcHx3QYchgyut\nLkWE4Lx4K5caOvLkyx8Wu76U5ER8kv/hjGNHzC0s9RChopQNpmbmBI2ZT5j3ezRMPcCNb9ty7cr5\nvH0/hkzAd/N2ojp747nzHKe7dmbXL1NV4sRSQJeV40FCiINCiBQhRKYQIqci7seRk5PNmXfeIMdE\n4D9jbrEfUQFE7lxOJZGJXaAaFFcqphYDJ3Kq3Xyq5lzH9JcOnM3fVdDOwYWes/9E/DKDlCqWVJux\njC19WhJ1cr+RI67YdPmt9z3wLHkL8CoBw/PPVSg7Zr1DrQt3uP1qH6p7NtRLnRaRfxArnKkf0FEv\n9SlKWeTXth8Jg9aRKcypueYZDm9ccP8znyd60nZDCDEjulP1UhJ3Br7MhskvkpZa4f52LRV0+nNZ\nSnkeMJFS5kgpF/D/SQorhMunD+K0YCOXvKvQduRUvdR589plfNMOEV2jOxoTE73UqShllYd3ABYj\ndxJtVoemYeMIWfQeMv/RlKmZOZ3Gf43b2j+50rQGnqvCONipFQf/LnqKH6VodOk4UoUQ5sBRIcR0\nIcSbgLWB4ip1cnNzOfXOWKSAxl/p5xEVwPkdizERkhqtVCZcpWxZ0HUBC7ouKLygjhyrueHx1nbC\nbTsQHDWH8FnPkpH+/ynZq7l702PJdu58+SZCSmwmfM26oR25EXNW77EoBdPlt9/z+eVfA+6Sl6W2\nnyGCKo12fP8u7mdvc2v409So46e3eh0uruGCSW3cvZvprU5FKessK1nT7M1VhNQaQfPbm7jwdScS\n4+P+VSaw1wiabdlLVL/m1Dp0lSs9erPt2wkqcWIJ0GVWVTR5SQarSyk/kVKOz390Ve5dPX8Uh/lr\nia5nT/vXPtdbvVfOn6Be9lniPZ/WW52KUl4IjYbgl2YQHjCDOplnuPtDWy6f/XfyCCubynT/dDHW\ny+cRX8sO17nr2NGtBZGhG40UdcWgy6yqp4CjwKb89/4VYQFgbm4uJ94ejZDg+9UcvT2iAojZvYhc\nKajd7kW91ako5U1AzxFE9VyBlUyl8rLunPzn4V87dfxa03nNfq69NQi7hDRyh41n3bhnuJNUYNLt\nfzkVl8ypODXIrgtdfgt+DAQCSQD5eaM89B9S6bJr7ge4n07k5rBu1Kynv8dJMjeXmjHrOGXZGGfX\noqdhV5SKoEHzjqS/uI1bGkfqb32RA3/MfKiMRqOh3SsfUW/TZqLaeFFnUwQnOrdn37Jv1NoPPdOl\n48iWUt42WCSlUFzUSex/Ws3lOra0f6M4GeQfdu7oHtxkHGn1++q1XkUpr2p41Mfh9V1EVmpC4ImP\nCZ07mpzsh8czqlStSc+5f5P5wydkWJniMOVnNg1ow5Wzh4wQdfmkS8dxUggxGDARQtQVQnxH3p4c\n5VJubi5HJ4zEJFfSYMZ3mJgUuj27ThJDlpAhzajfXu2/rCjasqvsiM9bGwlz6kvQtaUc/+Yp7t5J\nKrBs4/YDaLl5P5dfaEf1MzdJ6DuETVNHkJmRWmB5RXu6dBxjAV8gg7xtXJOBcYYIypiGbRrGsE3D\n2PPLFDxOJnDj+U64+7TQaxvZWZl4xW8lwiYYu8qOeq1bUco7UzNzWry2gND6E/G7G8K1b9txPeZC\ngWXNLazo8u4POK9eTqyvM+5L97K/czBHty4v4ahLxr3fX4amy6yqVCnl+1LK5uRtvDRZSlkucx6b\nJ97FZs5Krnja0OGtb/Re/6l/1uLIbfAboPe6FaWiCHr2PU62mUe17DjE/I6cO7r3kWVdvfzp/vtu\nEj8ZiVl6DhZjp7Dule7cuhbNpGWXmLTsUskFXg4U2nEIIT4UQjTIf20hhNgBnCdvz/BylyND5ubS\ndsUZzLIlXtNn6v0RFUDmkRUkY41vmwqzDEZRDKJx+wHED1hDLia4ru7H4c2P35T0iYFv0HjLLi72\n8MPjnyjOd+9GklUmeVsOKdrS5o5jIHAm//UL+dc4A22AzwwUl9HUDLuM/5lMrj3XjtqNWum9/tSU\n2/jc3sPpKu2wsLTSe/2KUtF4+rbAdOROrph54L9/LKGLP7ifpqQgNvZO9Pj6d0wXfcttZyuCzmRj\nnp3B+WMVMtl3kWjTcWRKKWX+6y7A8vxcVZGA/v8cN6IbMWd5as01op0FHd7+1iBtnNr1O1YiA+vm\nzxmkfkWpiJxcauI+fgdH7NoSdHE2B2c/R2bG45+kNwjsQvv1oYTWN8UlUZI6eCQbPhxGZpoaPC+M\nNh1HhhCioRCiKtAO2PLAZ+XqT2YrW0cuuJmQ7GCJqZm5Qdowi1jFNZzwbtHFIPUrSkVlaWVDk3F/\nEOr2MoFJGzj3dSduJ1x/7DUmJqZUTjXnbA1LrvhXx3NlKPu7BnNsx8oSirps0qbjGAesAk4DM6WU\nUQBCiO7AEQPGVuJs7B3Z8lpzNo5sbJD6b924im/qQaKqd1WZcBXFADQmJgQN/4bwpl9SN+MUyd+3\n5cr5E4VeVylbQ4+lO0iaOhqz9BxMR3/EutFPk3zrWglEXfYU2nFIKUOllA2klI5SyqkPnN8gpXzW\nsOGVL+d2LsFU5FKtpcqEqyiGFPD0SC52X46NTMHuty5E7Fv/yLJfDPbgi8EeAAT3H4vf5h15uw7u\nOMfJrh3Yt+LhVeoVnTazqsY/7iiJIMsL+3N/EaVxp3ZD/a4LURTlYQ1adCZ16BaSNA7U3fI8B1bP\n1uo628rO9Jz9J7lzp5FhZYbDx/NY/2xbrkdHGjjiskObR1W2+UcAMApwzT9GAj6GC618uXoxkgbZ\nkVxzf8rYoShKheFa25vKY3dxxrIxgcc+IOSnMeTm5Gh1baO2/Wi5eT+XBrXE7fh1Yp7ux7bZE1Xa\ndrR7VPWJlPITwAloKqV8S0r5FtAMcDN0gCXNUJvTXN69EACPtuoxlaKUJPsqTjR4axNhjr0JjvuN\nY988TWqKdmn3zC2s6PbxfOx+/4WbtWxx/WEt2596osJP3dUl5UgtIPOB95lUgOy4+iBzc6lxZR2n\nzBtR3b2+scNRlArHzNyCwDELCK03Ab+UfVyd2Z742EtaX+/Z8Ak6rQkh7vW+OMSm5E3d/eDFCjt1\nV5eOYwlwQAjxsRDiIyAMWGyYsMqX88f34Z4bw526fYwdiqJUWEKjIWjwZE62mUuN7BjkvHacP7ZP\n6+s1Gg3tR39KrXV/5e15/r8w9ncJ5tj2ijd1V5dcVZ8Cw4BE8vbkGCalLHcrxw0hIeQ3MqUJDVQm\nXEUxusbtB3G9/1py0VDjzz50ib+Fe1bBSRIL4uxWjx5LtnN72muYZeRgOiZv6u7thLjCLy4ntJlV\nZZf/1QG4RN6dxxIgOv+c8hg52dnUub6ZCOsW2DtWM3Y4iqIAtRu2wHTEDmLM3Pn47hGejK+icx1B\nz4yh8ZZd96funurakX3L9Z8UtTTS5o5jWf7XQ0D4A8e998pjnNq/nqokkttQZcJVlNLEqYY7tcbv\nZJumPq9lHSbk53GPzXFVEBt7p7ypuz99RpqNGQ6f/MyGQW25Fn3KQFGXDtrMquophBBAGyll7QcO\nTyll7RKIsUxLP7ycFFkJ37b9jR2Koij/YWllw/xqGawWDQm+uoCwea/p3HkANGrTh1ab9nNpcCtc\nT17n6tP92PbthHI7dVerMY78JIerDRxLuZOemoJ34i4iq7TF0srG2OEoilIQDSyvdvf+roJhc18t\nUudhbmFFtw9/xm7FL8TXssd17jq29wzm/JFd+o/ZyHSZVRUqhGhusEjKoYhd/8NGpFGpmcrMoiil\nVbRZHaIt6hA4+hdCnQcSdGMlB+YM03qh4H95NnyCzmv2E/dGPxzi7pI6ZBQbJr9IRlqKniM3Hl06\njnZAiBDighDiuBDihBDiuKECKw80J1cSTxW8g3sYOxRFUQohNBpajJxLSPWhtEj4i/Dvhxa589Bo\nNLQfNY1a6/7icjNXPFeFEdKlJUe3r9Bz1MahS8fRDagDtAeeAnrmf1UKcDvhOr53w7hQrSsmpuVq\n2xJFKbeERkPQK7PyUrMnruPQ7GfJyS76OIWzWz16Lt7G7U/zpu5ajPmEdaOeKvNTd3VZxxEtpYwG\nsgGZf1w1VGBl3ekdSzAXOTg9odZuKEpZIjQagoZ/Q4j7SJrf3syRWQPIzsos/MLHCOqXN3X3Qhcf\nPHedz5u6u6zsTt3VZh3Hu0KIDx84FQKsI29DpwmGCqyssz27mmiNG3UaPWHsUBRFKYLgYV8SUvt1\nAu5s5/isfmRlZhSrPht7J3rO+gM57/O8qbtTfmbDoDZlcuquNncc/YGvH3ifIKX0A3wB9fC+AHHR\nZ/DJOklszZ4IjS5PAxVFKU2Ch04ltO5bNE3Zw8lve5ORXvzcVA1b96bVpv1EP9ca15M3yuTUXW2n\n49594O2s/HM5QKWiNiyE6C+EiBBC5AohAh5TrqsQ4owQ4rwQYlJR2ytJl3blpfCq1eZF4waiKEqx\nBT33IWHe79IkdT+nZ/UiPe1u4RcVwtzCiq4fzMP+91+Jd39w6u5OPURseNp0HDZCCLN7b6SUCwGE\nEBaAXTHaPgn0BfY8qoAQwgSYQ97AvA/wrBCiVO8BInNzqR69ltNmPrjW9jZ2OIqi6EGLgZMI8/2Q\nRqkHOfdtT9Lu3tFLvR6+wXT+az9x456hyrW7pD03mg3vv1Dqp+5q03GsAn4SQljdOyGEsAbm5n9W\nJFLKSCnlmUKKBQLnpZQXpZSZwAqgV1HbLAkXIw7gkXuZ2169jR2Koih61KL/W4T7T8U3/QgXZ/XQ\nek+Pwmg0GtqPnIrHurVEB7ji+ceBvKm7W5frpX5D0Kbj+AC4AVwWQhwSQhwiL9nh9fzPDMkVuPLA\n+5j8c6VW/L7FZEkT6rVXGzYpSnkT2Gcsh5t9QYOM41ya1Z2U5ES91V3V1Stv6u5nYzHLzMFi7JRS\nO3VXm1xVOVLKSUBN4MX8o5aUcpKU8rGjOUKIbUKIkwUc2t41iIJCekRbI4QQ4UKI8Pj4eC2r16/c\nnBw8r23ilFUAVapWN0oMiqIYVsDTIzna4mvqZZ4iZnY3kpMS9Fp/UN/RNN68iwvdfO9P3f1n6Vd6\nbaO4dJny8xJwRUp5QkqZJoSoIoQY/bgLpJQdpZQNCzjWaNlmDHkd1j1uQOwj2ponpQyQUgZUrVpV\ny+r1KzJ0E9VIIMv3GaO0ryhKyWjW/WWOPzGb2llnufZ9V27f0u8fqzb2TvScuQp+/oI0W3Mcp/7C\nhoFtiIs6qdd2ikqXjuMVKWXSvTdSykTgFf2H9C8HgbpCCE8hhDkwCFhr4DaL7G74MlKlBT5tBxo7\nFEVRtOSR+TYemW/rfF3TLs9zqvUcPLIucnNOZxLj9f9IybdVL1pt3Ef0kCdxjbhBbO/+bJ35ttGn\n7urScWjy06sD92c8mRe1YSFEHyFEDBAMrBdCbM4/X0MIsQEg/1HYa8BmIBJYKaWMKGqbhpSedpcG\niTuIqNwGKxt7Y4ejKEoJ8O/4LKfb/oRr9hWSfuxKwvUYvbdhbmFF18k/UXnlQuLd7XH7aT3bewQZ\ndequLh3HZmClEKKDEKI9sBzYVNSGpZSrpZRuUkoLKWU1KWWX/POxUsruD5TbIKWsJ6Wsk799bakU\nuWcVdqRi0WSQsUNRFKUE+bV7hvMdf8ElJ5aUn7pyMzbaIO24+7Sg81/7uTZ+AFWup+ZP3R1qlKm7\nunQcE4EdwChgDLAdeMcQQZVF8vj/SMAen5Yq76OiVDQNW/fiYpdFVM25QdrPXblxNcog7Wg0GtqN\n+ATP9X8T3dwNzz8OEtr5CY5sWWqQ9h4Zh7YFpZS5UsofpZTPSCn7SSl/yl89XuHdTrxJw5QQzlXt\njKlZkZ/eKYpShvk+0Z3L3X+jSm4iWfO7EBdd2DK1onOqUYeei7aS/PkbmGTnYvn6NNa92pM+c04w\n6DvDP83XJsnhyvyvJ/L34fjXYfAIy4AzO37DXGTjEKwy4SpKWfP7q8H8/mqwXupq0KIzV59ahq28\nAwt6EBt1Wi/1PkqLPiPx37Q7b+rungtUjb3LDfMMcouwg6EutLnjeCP/6739N/57VHjWZ/4kRlSn\nrv+Txg5FURQjqx/Qnuu9V1KJNEwWdSfmvGGn0NrYO+ZN3Z3/JSmVwCU+k1wDz7rSZgHgvTlmo+/t\nyfHA3hyPXcdREdy4GoV3xnGuuKlMuIqi5Knr35qEfn9gThYWv/Uk+sxRg7fp2/JpbleyJN7O0uCP\nzHX5TdepgHPd9BVIWXVx50I0QuLW5gVjh6IoSilSp1EQtweuRiCxXt6LqFMHDd6mCRoqZRn+D1ht\nxjhGCSFOAA3+M74RBVT4MY6qUWs4a1qPml6NjB2KoiiljId3AHefXYNEUHllXy6cCDVoe18M9uCL\nwR4GbQO0u+NYRt5Yxhr+PbbRTEpZoUeDL0WGUycnilt1VCZcRVEK5l7fn/Qhf5OJGU5/9OXc0b3G\nDqnYtBnjuC2lvARMBq7lj214AkOEEJUNHF+pFrd3MdlSg1c7lQlXUZRHq+nViJwXNpCKFdX+GsCZ\n8B3GDqlYdHkY9geQI4TwAn4hr/NYZpCoyoDcnBw8YjdwqlJTnFxqFn6BoigVWg3PBjBsPXeELa5/\nD+Z02BZjh1RkunQcufm5o/oC30op3wQqbO7wM+HbqE48Gd79jB2KoihlRHX3+pgN30yipgq1Ngwh\nYv8GY4dUJLp0HFlCiGeBocC6/HNmjylfriUfyM+E236wsUNRFKUMcXb1pNIrm4g3cab25hc4uVfb\nXSZKD106jmHkZbL9VIzCMoEAAAxTSURBVEoZJYTwBH4zTFilW2ZGOvUTtnHKrhXWthV6mEdRlCJw\nquGOzaubuGZSA69tL3N8Z5F34TYKXXJVnZJSvi6lXJ7/PkpK+YXhQiu9Tu35k8qkYNZkgLFDURSl\njHKs5kblUZu4alqTBrte5ej2FcYOSWsqV1UR5Bz7nUTs8GnVx9ihKIpShlWpWh2n0ZuINvPEZ89o\nDm9eYuyQtGKqRZkHc1VVeHdu38L3zj6OVX2KFuYWxg5HUZQyzt6xGuK1zUR93x2//a9zKCeTZt1f\nNnZYj6V1rqr/5ql6IF9VhXJ65zIsRRb2gWpQXFEU/bCr7Ijr65s4Z+6Nf9hbhK/90dghPZbWYxxC\niDtCiOT/HFeEEKuFELX/r727D7KqruM4/v7s8rAIWCRbyINsImFSqIU8pFmmGWmplWYNWo42ZpOF\nNZaZlaNNNY1jT1ajWFimIRWiKTqVSZENC6JCAou10KCEgOggUios++2Pe7DVlmXPfdjf3buf18yd\nufdy7jmf3wD3O+ec3/3+KhmymjS0zGeTXseEySemjmJmNWTIgcMYO+te1g6cxFseupxlC65LHWmf\n8syq+g7wBWAUMBq4FLgRuA2YU/5o1Wfb5sc54oVH2DDqVHfCNbOyO2DIqzh01kJWNxzNlJVfYemv\nr00dqVN5vv1mZKv+PRcROyJiNnBKRMwDhlUoX1Vpvf/n1CsY+XZ3wjWzyhg0eCjjL7mblYOmMHX1\n1SydV32TV3P9clzShyXVZY+Oc1Gj3MGq0UHr7qC1fhxjJxyVOoqZ1bCGQYM5fNadPHLA25ja8i2a\nb706daSXyVM4ZgLnAluzx7kUGh0OAi6uQLaq8vjfVzB+TyvbDj09dRQz6wMGNhzAxFkLeHjw8Uz7\nx7UsufmrqSO9JM8PANdHxPsjYnj2eH9EtEbE8xHxQCVDVoNNi2+mPcS4E3yZysx6xoCBDUy6ZD7L\nh57I9PU/YMlNl6WOBOSbVTU6m0G1VdIWSfMlja5kuGoR7e2M+dfdrGk4ksaRTanjmFkf0q//AI6e\n9SsefNXJTN9wPUt+8jmivT1ppjyXqm4CfguMpDCz6q7svZr32MOLGBVb+M/h7oRrZj2vvl8/3vKZ\nuSwbdirTN86h+cbPJC0eeQpHY0TcFBFt2eNnQGOFclWVZ5tv5YXoz+EnzEwdxcz6qPp+/Zh88S9Y\netAZTH/yFpZef1Gy4pGncGyTdI6k+uxxDvB0pYJVi927XuQN2/7AmqFv48BXH5Q6jpn1YXX19Uz5\n9E00N57FtK3zWPbjC2jfs6fnc+TY9nzgw8Bm4EngTAqt1mvamgfuZBg70CR3wjWz9FRXx9RPzaZ5\nxEymbrud5T/6eI8Xjzyzqh6PiNMiojEiXhsRZ1BYDbCm7V4xj2cZzMR3nJk6ipkZkBWPC3/IklHn\nMeWZu3joupnsaWvrseOX2jfj82VJUaX+/dx2jnj2L6x9zYkMGNiQOo6Z2UtUV8e0C77LkkMu5Jjt\n9/LID86m6cX1jN29ruLHLrVwqCwpqlTLots4QC8y9Bh3wjWz6qO6Oqaffw3NTZ9m8o77uGDLANRe\n+a/lUgtHTbca6b/mN2ymkcOnnJw6ipnZPk0775s0H3YJJ8daPrm5P227d1X0eN1ZAbCzduo7JD1H\n4TcdNenpLRuZ+PxD/HPke6mrr08dx8ysS9POuYrvD3grm+obqK/vzhp9xdvv3iNiaEUTVKl/3H8z\n09TOiOM+ljqKmVm3/HX4MwCcU+FlH7yoxD4MW3cH6+uaeP0Rx6SOYmZWVVw4OrGxdRUT2h5ja9Np\nqaOYmXXbhv7j2NB/XMWP48LRiSeyTrhN7oRrZvZ/XDheIdrbGf3EXbQMfBMjxhyWOo6ZWdVJVjgk\nnSVptaR2SZO72G5O1sp9VU/kal35AGNiE/+e4E64ZmadSXnGsYpCy5LF+9nuZ8CMiqfJPL3kFnZF\nPya869yeOqSZWa9S2cm+XYiIFgCp6185RsRiSU09EIm23bs4bOvvWD1kGkcPG94ThzQzK5umXZf2\nyHF8j6OD7U89yeaBTcSkj6SOYmZWtSp6xiHpPmBEJ390RUTcWeZjXQhcCHDIIYcUtY/hI8cy/PI/\nlzOWmVnNqWjhiIiTKrn/VxxrNjAbYPLkyTXdQ8vMLCVfqjIzs1xSTsf9gKSNwHRgoaTfZe+PlHRP\nh+3mAkuACZI2SrogTWIzM4O0s6oWAAs6eX8TcEqH1x/tyVxmZtY1X6oyM7NcXDjMzCwXFw4zM8vF\nhcPMzHJx4TAzs1xcOMzMLBcXDjMzy8WFw8zMcnHhMDOzXFw4zMwsFxcOMzPLxYXDzMxyceEwM7Nc\nknXHNTOz8pr3yek9chyfcZiZWS4uHGZmlosLh5mZ5eLCYWZmubhwmJlZLi4cZmaWiwuHmZnl4sJh\nZma5uHCYmVkuiojUGcpO0lPAhhJ2MRzYVqY4KdXKOMBjqVa1MpZaGQeUNpaxEdG4v41qsnCUStLy\niJicOkepamUc4LFUq1oZS62MA3pmLL5UZWZmubhwmJlZLi4cnZudOkCZ1Mo4wGOpVrUylloZB/TA\nWHyPw8zMcvEZh5mZ5eLC0QlJX5f0N0krJP1e0sjUmYol6RpJa7PxLJD06tSZiiXpLEmrJbVL6nUz\nYCTNkPSYpFZJX0qdpxSS5kjaKmlV6iylkDRG0iJJLdm/rVmpMxVLUoOkZZJWZmO5qmLH8qWq/yfp\nwIjYkT3/LHBERFyUOFZRJJ0M3B8RbZK+DRARlyWOVRRJbwTagRuASyNieeJI3SapHvg78G5gI/Ag\n8NGIWJM0WJEkHQ/sBG6OiDelzlMsSQcDB0fEw5KGAg8BZ/TGvxdJAgZHxE5J/YEHgFkR0VzuY/mM\noxN7i0ZmMNBrq2tE/D4i2rKXzcDolHlKEREtEfFY6hxFmgK0RsT6iNgF3AacnjhT0SJiMfBM6hyl\niognI+Lh7PlzQAswKm2q4kTBzuxl/+xRke8uF459kPQNSU8AM4Gvpc5TJucD96YO0UeNAp7o8Hoj\nvfQLqlZJagKOBpamTVI8SfWSVgBbgT9EREXG0mcLh6T7JK3q5HE6QERcERFjgFuBi9Om7dr+xpJt\ncwXQRmE8Vas7Y+ml1Ml7vfZMttZIGgLMBy55xRWHXiUi9kTEURSuLEyRVJHLiP0qsdPeICJO6uam\nvwQWAldWME5J9jcWSR8H3gecGFV+UyvH30tvsxEY0+H1aGBToizWQXY/YD5wa0TcnjpPOUTEdkl/\nAmYAZZ/A0GfPOLoiaXyHl6cBa1NlKZWkGcBlwGkR8Z/UefqwB4Hxkl4vaQDwEeC3iTP1edkN5Z8C\nLRHxndR5SiGpce+sSUmDgJOo0HeXZ1V1QtJ8YAKFGTwbgIsi4l9pUxVHUiswEHg6e6u5F88Q+wBw\nHdAIbAdWRMR70qbqPkmnAN8D6oE5EfGNxJGKJmku8E4KnVi3AFdGxE+ThiqCpOOAvwCPUvj/DvDl\niLgnXariSJoE/JzCv6864FcRcXVFjuXCYWZmefhSlZmZ5eLCYWZmubhwmJlZLi4cZmaWiwuHmZnl\n4sJhViRJO/e/VZef/42kQ7PnQyTdIGld1tl0saSpkgZkz/vsj3Wt+rhwmCUgaSJQHxHrs7d+QqFp\n4PiImAicBwzPGiL+ETg7SVCzTrhwmJVIBddkPbUelXR29n6dpB9nZxB3S7pH0pnZx2YCd2bbjQOm\nAl+JiHaArIvuwmzbO7LtzaqCT3/NSvdB4CjgSAq/pH5Q0mLgWKAJeDPwWgotu+dknzkWmJs9n0jh\nV/B79rH/VcAxFUluVgSfcZiV7jhgbtaZdAvwZwpf9McBv46I9ojYDCzq8JmDgae6s/OsoOzKFhoy\nS86Fw6x0nbVM7+p9gOeBhuz5auBISV39fxwIvFBENrOyc+EwK91i4OxsEZ1G4HhgGYWlOz+U3et4\nHYWmgHu1AIcBRMQ6YDlwVdatFUnj965BIukg4KmI2N1TAzLriguHWekWAH8DVgL3A1/MLk3Np7AO\nxyoK66QvBZ7NPrOQlxeSTwAjgFZJjwI38r/1Ok4Ael23Vqtd7o5rVkGShkTEzuysYRlwbERsztZL\nWJS93tdN8b37uB24vBevt241xrOqzCrr7mxxnQHA17MzESLieUlXUlh3/PF9fThb9OkOFw2rJj7j\nMDOzXHyPw8zMcnHhMDOzXFw4zMwsFxcOMzPLxYXDzMxyceEwM7Nc/guHVaRDwqV2nwAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x110172910>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差\n",
    "test_means = grid.cv_results_['mean_test_score']\n",
    "test_stds = grid.cv_results_['std_test_score']\n",
    "train_means = grid.cv_results_['mean_test_score']\n",
    "train_stds = grid.cv_results_['std_train_score']\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs, number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs, number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs, number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs, number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr = test_stds[:,i], label = penaltys[i]+'Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr = train_stds[:,i], label = penaltys[i]+'Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg_logloss')\n",
    "pyplot.ylabel('LogisticGridSearchCV_C.png')\n",
    "pyplot.title\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression + L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.01, 0.1, 1, 10], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "Cs = [0.01, 0.1, 1, 10]\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv=5, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.27959842, -0.26051579, -0.25584139, -0.26672131],\n",
       "        [-0.27959842, -0.24974326, -0.24281703, -0.26465216],\n",
       "        [-0.27986375, -0.26458756, -0.28592529, -0.32145597],\n",
       "        [-0.27986375, -0.3157092 , -0.32590428, -0.33518429],\n",
       "        [-0.27986375, -0.26661279, -0.2585213 , -0.26918116]]),\n",
       " 1: array([[-0.54366306, -0.50735628, -0.55351782, -0.63042246],\n",
       "        [-0.54370493, -0.51547757, -0.57977205, -0.6920664 ],\n",
       "        [-0.54178104, -0.48012698, -0.50803942, -0.59453833],\n",
       "        [-0.54224855, -0.50474151, -0.54964758, -0.61453274],\n",
       "        [-0.54243667, -0.49171544, -0.52382175, -0.57392036]]),\n",
       " 2: array([[-0.61085861, -0.56089459, -0.59288752, -0.65435114],\n",
       "        [-0.61105993, -0.56789486, -0.63739445, -0.7264703 ],\n",
       "        [-0.61162003, -0.55349989, -0.59366648, -0.71789735],\n",
       "        [-0.61342577, -0.6194761 , -0.68278469, -0.74504339],\n",
       "        [-0.60838268, -0.55533056, -0.60144487, -0.6759598 ]])}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VNX5x/HPwxL2nci+70uCQAQU\npFg3RAVZVFSsVK2IUqp1AavViloVd+uGWFuLiiKIbAoKKqgUJAESwh5AIaxhXwKEJOf3xww2PwzM\nkGRyk5nv+/WaV+beuXfmuTNJnjnn3Pscc84hIiJyJiW8DkBERIo+JQsREQlIyUJERAJSshARkYCU\nLEREJCAlCxERCUjJQkREAlKyEBGRgJQsREQkoFJeB1BQatas6Ro3bux1GCIixUpCQsJu51x0oO3C\nJlk0btyY+Ph4r8MQESlWzOznYLZTN5SIiASkZCEiIgGFNFmYWW8zW2tmKWY2OpfHh5pZmpkt999u\nP+Xxyma21cxeC2WcIiJyZiEbszCzksDrwKVAKrDEzKY751adsunHzrkRp3maJ4D5oYpRRESCE8qW\nRRcgxTm30TmXAXwE9At2ZzPrDNQCvgxRfCIiEqRQJot6wJYcy6n+dacaaGZJZjbZzBoAmFkJ4AXg\ngRDGJyIiQQplsrBc1p06Ld8MoLFzLhaYC7znX38X8LlzbgtnYGZ3mFm8mcWnpaXlO2AREcldKK+z\nSAUa5FiuD2zLuYFzbk+OxfHAs/775wMXmtldQEUgyswOO+dGn7L/28DbAHFxcZofVkQizqyk7WQ5\nR98OdUP6OqFMFkuAFmbWBNgKDAZuzLmBmdVxzm33L/YFVgM4527Ksc1QIO7URCEiEunW7DjI/Z8k\n0q5uZa6KqUOJErl16BSMkCUL51ymmY0A5gAlgXedcyvNbAwQ75ybDow0s75AJrAXGBqqeEREwsmB\noycYNiGBimVL8cZNnUKaKADMufDovYmLi3Mq9yEikSA723H7f+JZsC6Nj+7oRlzj6nl+LjNLcM7F\nBdpOV3CLiBQzr369nq/X7OLRq9vmK1GcDSULEZFiZN7qnbw8dz0DOtXj5m6NCu11lSxERIqJn3Yf\n4Z6Pl9O2TmX+3j8Gs9COU+SkZCEiUgykZ2QybEICJUsY427uTNnSJQv19cNmPgsRkXDlnGPUlBWs\n23WI937fhQbVyxd6DGpZiIgUcf/8fhMzErdx/2Wt6Nky4KR2IaFkISJShP13wx6e/mINl7erxV29\nmnkWh5KFiEgRtf3AUUZ8uJRGNcrz/LUdCnVA+1QasxARKYKOZ2Yx/P2lHDuRxcc3d6NS2dKexqNk\nISJSBD0+YxXLt+znrSGdaH5OJa/DUTeUiEhR8/GSzXy4eDPDezWjd/s6XocDKFmIiBQpiVv289dp\nK+nRvCb3X9bK63B+oWQhIlJE7Dl8nOHvJxBdsQyv3tCRkiGuJHs2NGYhIlIEZGZl88eJy9hzJIMp\nwy+geoUor0P6f5QsRESKgOfmrGXhhj08NyiW9vWqeB3Or6gbSkTEY7OStjNuwUaGdGvItXENAu/g\nASULEREPrdt5iAcmJ9KxYVUevaqd1+GclpKFiIhHDh7zTY1aPqoUb97UmahSRfdfctGNTEQkjGVn\nO/78cSJb9qbzxk2dqF2lrNchnZGShYiIB17/JoW5q3fy8JVt6NKkcKZGzQ8lCxGRQvbN2l28OHcd\n15xbl6EXNPY6nKAoWYiIFKKf9xzhTxOX0bp2ZZ4eEOtpJdmzoWQhIlJIjmZkMWxCAgDjhnSmXFTh\nTo2aH7ooT0SkEDjnGP1pEmt3HuLdoefRsEbhT42aH2pZiIgUgn8v/Ilpy7fx50taclGrc7wO56wp\nWYiIhNjijXt4atZqLmlTi7svau51OHmiZCEiEkI7Dhzj7g+X0aB6eV68vgMlilAl2bOhMQsRkRA5\nnpnF8A8SSM/I5MM/dKWyx1Oj5oeShYhIiDwxcxXLNu/n9Rs70bKW91Oj5oe6oUREQmBS/BbeX7SZ\nYT2bcmVs0ZgaNT+ULERECtiK1AM88lkyFzSrwQOXF52pUfNDyUJEpADtPZLBne8nULNCFP+4oSOl\nSobHv1mNWYiIFBDf1KhLSTt8nMl3nk+NimW8DqnAhEfKExEpAp7/ch0/pOzhyX7tia1f1etwClRI\nk4WZ9TaztWaWYmajc3l8qJmlmdly/+12//pzzey/ZrbSzJLM7PpQxikikl9frNjOW/M3cGPXhlx3\nXtGcGjU/QtYNZWYlgdeBS4FUYImZTXfOrTpl04+dcyNOWZcO/M45t97M6gIJZjbHObc/VPGKiOTV\n+p2HuP+TRM5tUJXHrm7rdTghEcqWRRcgxTm30TmXAXwE9AtmR+fcOufcev/9bcAuIDpkkYqI5NHJ\nqVHLRZXkzSGdKFOq+FSSPRuhTBb1gC05llP960410N/VNNnMftV2M7MuQBSwITRhiojkTXa2475J\nify8N53XbuxEnSrlvA4pZEKZLHIrgOJOWZ4BNHbOxQJzgff+3xOY1QEmAL93zmX/6gXM7jCzeDOL\nT0tLK6CwRUSC8+b8DXy1aid/6dOGbk1reB1OSIUyWaQCOVsK9YFtOTdwzu1xzh33L44HOp98zMwq\nA7OAR5xzi3J7Aefc2865OOdcXHS0eqlEpPDMX5fG81+upW+HutzavbHX4YRcKJPFEqCFmTUxsyhg\nMDA95wb+lsNJfYHV/vVRwFTgP865T0IYo4jIWduyN50/fbSMVrUq8czAmGIzNWp+hOxsKOdcppmN\nAOYAJYF3nXMrzWwMEO+cmw6MNLO+QCawFxjq3/06oCdQw8xOrhvqnFseqnhFRIJxcmrU7GzHuJs7\nUz4qMq5tNudOHUYonuLi4lx8fLzXYYhIGHPON6A9dflW3r3lPC5qXfxmvDuVmSU45+ICbacruEVE\ngjRh0c98umwr91zcMiwSxdlQshARCUL8T3sZM2MVF7c+hz/+tnhOjZofShYiIgHsOniM4R8spX61\ncrx4/bnFdmrU/IiMkRkRkTzKyMzmrg+WcvhYJhNu60KVcsV3atT8ULIQETmDp2atIv7nfbx6Q0da\n167sdTieUTeUiMhpTElI5b3//sztPZrQt0Ndr8PxlJKFiEgukrce4C9TV9CtaXVGX9Ha63A8p2Qh\nInKKff6pUauVj+K1GzuFzdSo+aExCxGRHLKyHSM/Wsaug8f5eFg3aobR1Kj5oWQhIpLDi1+t5bv1\nu3l6QAwdG1bzOpwiQ20rERG/2ck7eP2bDQw+rwE3dGnodThFipKFiAiQsusw93+SSIf6Vfhb33Ze\nh1PkKFmISMQ7fDyTYRPiKVOqBG8O6UzZ0uE5NWp+aMxCRCKac477JyWyafcR3r+9K3Wrhu/UqPmh\nloWIRLS35m9k9sodPHRFGy5oVtPrcIosJQsRiVjfrU/juTlruDK2Drdf2MTrcIo0JQsRiUhb9qYz\ncuIymp9TkbEDYyNiatT8ULIQkYhz7EQWwz9IIDPLMe7mOCqU0fBtIHqHRCSiOOd4eGoyyVsP8s7v\n4mhSs4LXIRULalmISER5f/FmpixNZeTFLbikbS2vwyk2lCxEJGIk/LyXMTNWclGraO65uIXX4RQr\nShYiEhF2HTrG8PeXUqdKOV6+vmNETo2aH0oWIhL2TmRlc/cHSzl47ATjbu5MlfKROTVqfmiAW0TC\n3lOzVrPkp328Mvhc2tSJ3KlR80MtCxEJa1OXpfLvhT9xa/cm9Du3ntfhFFtKFiIStlZuO8BDn66g\nS5PqPNRHU6Pmh5KFiISl/em+qVGrlCvN6zd2orSmRs2XoN49M+tuZhX894eY2Ytm1ii0oYmI5I1v\natTl7DhwjDeHdCa6kqZGza9gU+2bQLqZdQAeBH4G/hOyqERE8uHluetYsC6Nv/VtRydNjVoggk0W\nmc45B/QDXnHOvQJUCl1YIiJ58+XKHfzj6xSui6vPjZoatcAEe+rsITN7CBgC9DSzkoBOVBaRImVj\n2mHum5RIbP0qjOnXXpVkC1CwLYvrgePAbc65HUA94LmQRSUicpaOHM9k2IQESmtq1JAIumWBr/sp\ny8xaAq2BiaELS0QkeM45HpycxIa0w0y4rSv1NDVqgQu2ZbEAKGNm9YB5wO+Bf4cqKBGRszH+u43M\nWrGdUb1b0725pkYNhWCThTnn0oEBwD+cc/2BdqELS0QkOAtTdvPMF2voE1ObO3o29TqcsBV0sjCz\n84GbgFn+dQE7BM2st5mtNbMUMxudy+NDzSzNzJb7b7fneOwWM1vvv90SZJwiEkG27j/KiInLaBpd\nkbGDOmhAO4SCHbO4B3gImOqcW2lmTYFvzrSD/4yp14FLgVRgiZlNd86tOmXTj51zI07ZtzrwGBAH\nOCDBv+++IOMVkTB37EQWw99PICMzm3E3d6aipkYNqaDeXefcfGC+mVUys4rOuY3AyAC7dQFS/Nti\nZh/hu07j1GSRm8uBr5xze/37fgX0RoPqIoJvQPvRackkpR5g3M2daRZd0euQwl6w5T5izGwZkAys\nMrMEMws0ZlEP2JJjOdW/7lQDzSzJzCabWYOz2dfM7jCzeDOLT0tLC+ZQRCQMTPxxC5PiUxlxUXMu\nb1fb63AiQrBjFuOAPzvnGjnnGgL3AeMD7JNb56E7ZXkG0Ng5FwvMBd47i31xzr3tnItzzsVFR0cH\nCEdEwsHSzft4bHoyPVtGc++lLb0OJ2IEmywqOOd+GaNwzn0LVAiwTyrQIMdyfWBbzg2cc3ucc8f9\ni+OBzsHuKyKRJ+3Qce56fym1q5Tl1cHnUlJToxaaYJPFRjP7q5k19t8eATYF2GcJ0MLMmphZFDAY\nmJ5zAzOrk2OxL7Daf38OcJmZVTOzasBl/nUiEqFOZGUz4sOl7EvP4K0hnalaPsrrkCJKsKcP3Ao8\nDnyKr4toAb4L807LOZdpZiPw/ZMvCbzrP5NqDBDvnJsOjDSzvkAmsBcY6t93r5k9gS/hAIw5Odgt\nIpHpmS/WsHjTXl66vgPt6lbxOpyIY75issVfXFyci4+P9zoMEQmBacu38qePljP0gsb8ra+uBy5I\nZpbgnIsLtN0ZWxZmNoNcBpZPcs71zUNsIiJBW739IKOmJHFe42r8pU8br8OJWIG6oZ4vlChERHJx\nIP0Ed76fQOWyvqlRo0ppalSvnDFZ+C/GExEpdNnZjns+Xsa2/Uf56I5unFO5rNchRbSgBrjNbAW/\n7o46AMQDTzrn9hR0YCIS2V6Zt55v1qbxRL92dG5U3etwIl6wZ0N9AWQBH/qXB+M7K+oAvlLlVxd4\nZCISseat3skr89YzsFN9hnRr5HU4QvDJortzrnuO5RVm9oNzrruZDQlFYCISmTbtPsI9Hy+nXd3K\nPNVfU6MWFcGOFlU0s64nF8ysC3CycldmgUclIhHpyPFM7pyQQMkSxluaGrVICbZlcTvwrplVxNf9\ndBC4zcwqAE+HKjgRiRzOOUZNSWL9rkO8d2sXGlQv73VIkkOwJcqXADFmVgXfhXz7czw8KSSRiUhE\n+ef3m5iZtJ0He7fiwhYqDFrUBFuivIqZvYhv/u25ZvaCP3GIiOTbwg27efqLNVzerhbDf9PM63Ak\nF8GOWbwLHAKu898OAv8KVVAiEjm27T/KHz9cRuMa5Xn+Wk2NWlQFO2bRzDk3MMfy42a2PBQBiUjk\nODk16vHMbMbdHEelsqW9DklOI9iWxVEz63Fywcy6A0dDE5KIRIrHZ6wkMfUAz1/bgebnaGrUoizY\nlsVw4L2TA9zkKCcuIpIXE3/czMQft3BXr2b0bq+pUYu6YM+GWg50MLPK/uWDIY1KRMLa8i37eWza\nSi5sUZP7LmvldTgShEAlyv98mvUAOOdeDEFMIhLGdh8+zvD3E4iuVIZXB3fU1KjFRKCWRaVCiUJE\nIkKmf2rUvUcymDL8AqpV0NSoxUWgEuWPF1YgIhL+np29hkUb9/L8tR1oX0+XahUnZz2TiJktDUUg\nIhLeZiRuY/x3m/jd+Y0Y1Lm+1+HIWcrLtFPqYBSRs7J2xyFGTUmic6NqPHJlW6/DkTzIS7KYVeBR\niEjYOnDUNzVqhTKleOMmTY1aXJ31p+aceyQUgYhI+MnOdtw3aTlb9qbzxk2dqKWpUYutYAsJHjKz\ng6fctpjZVDNrGuogRaR4eu2bFOau3sUjV7bhvMaaGrU4C/YK7heBbfimVTV806rWBtbiKzLYKxTB\niUjx9c2aXbw0dx39O9bjlgsaex2O5FOw3VC9nXPjnHOHnHMHnXNvA32ccx8D1UIYn4gUQz/vOcKf\nPlpG69qV+Xv/GFWSDQPBJotsM7vOzEr4b9fleMyFIjARKZ7SMzIZNiEBM2PckM6Ui9LUqOEg2GRx\nE3AzsAvY6b8/xMzKASNCFJuIFDPOOR76dAVrdx7ilcHn0rCGpkYNF8EWEtwIXH2ah78vuHBEpDj7\n1w8/MW35Nu6/rCW9Wp3jdThSgII9G6qlmc0zs2T/cqyZ6RRaEfnFoo17eOrz1VzathZ39WrudThS\nwILthhoPPAScAHDOJeE7I0pEhB0HjjHiw6U0ql6eF67rQAlVkg07wSaL8s65H09Zl1nQwYhI8XM8\nM4vhHySQnpHFuJs7U1lTo4alYK+z2G1mzfCf+WRmg4DtIYtKRIqNMTNWsWzzft64qRMtamlWg3AV\nbLK4G3gbaG1mW4FN+M6QEpEINmnJFj5YvJlhv2lKn5g6XocjIRRsstgK/Av4BqgOHARuAcaEKC4R\nKeKSUvfzyLRkujevwQOaGjXsBTtmMQ3fqbMn8JX9OAwcCbSTmfU2s7VmlmJmo8+w3SAzc2YW518u\nbWbvmdkKM1ttZg8FGaeIFII9h49z54QEoiv6pkYtVVKVZMNdsC2L+s653mfzxGZWEngduBRIBZaY\n2XTn3KpTtqsEjAQW51h9LVDGORdjZuWBVWY20Tn309nEICIFLzMrm5EfLWP3kQwm33k+NSqW8Tok\nKQTBfh1YaGYxZ/ncXYAU59xG51wG8BHQL5ftngDGAsdyrHNABTMrBZQDMvB1fYmIx577ci0/pOzh\nyWvaE1u/qtfhSCEJNln0ABL8XUpJ/u6hpAD71AO25FhO9a/7hZl1BBo452aesu9kfN1c24HNwPPO\nub1BxioiITIraTvj5m/kpq4NuS6ugdfhSCEKthvqijw8d25X5fxSdNDMSgAvAUNz2a4LkAXUxVfV\n9jszm+svO/K/FzC7A7gDoGHDhnkIUUSCNTNpGw9OTqJjw6o8erWmRo00wdaG+jkPz50K5PzqUR/f\n4PhJlYD2wLf+8sW1gelm1he4EZjtnDsB7DKzH4A44P8lC3+p9LcB4uLiVP1WJAT2Hcngr9OSmZm0\nnQ4NqvLWkM6UKaVKspEmlKcwLAFamFkTM4vCVx5k+skHnXMHnHM1nXONnXONgUVAX+dcPL6up9+a\nTwWgG7AmhLGKSC7mrd7JZS8vYM7KHTxweSum3Hm+pkaNUCFLFs65THzly+cAq4FJzrmVZjbG33o4\nk9eBikAyvqTzL389qlDEyWtfr2f7gaOheHqRYunQsRM8ODmR296Lp0aFKKbd3YO7L2quU2QjmDkX\nHr03cXFxLj4+/qz325h2mD6vfkepEiUYfUVrbuzSUEXQJKItTNnNA5OT2H7gKHf1as7Ii1sQVUpJ\nIlyZWYJzLi7QdhH/G9A0uiJz7ulJbP0qPPJZMoPHL2LT7oDXG4qEnaMZWfxt+kpufGcxZUqVYMrw\nC7j/8lZKFAKoZfEL5xyT4rfw5KzVZGRmc++lLbm9RxM1uyUiJPy8j/s/SWTT7iPc2r0JD1zeStOh\nRohgWxbBnjob9syM689rSK9W5/DotGSe+WINM5O28ezAWNrVreJ1eCIhcTwzi5fnrmfc/A3UqVKO\niX/oxvnNangdlhRBalmcxhcrtvPXaSvZl57BsJ5NGXlxC8qW1jctCR/JWw9w36RE1u48xA1dGvDw\nlW2pWEbfHyONWhb5dEVMHc5vVoOnZq3mjW83MDt5B88MjKVLk+pehyaSL5lZ2bzx7QZenbee6hWi\n+NfQ87iotebLljNTh/wZVC0fxXPXduA/t3YhIyub68b9l79+lsyhYye8Dk0kT1J2HWLgmwt58at1\nXBlbhy/v7alEIUFRN1SQjhzP5IUv1/GvhZuoXbksT/Vvz29b1wrZ64kUpOxsx7s/bGLsnLVUiCrJ\nU/1jNFmRAMF3QylZnKWlm/cxanIS63cdpt+5dXn0qrYq0SxF2uY96dw/OZEfN+3lkja1eHpADNGV\n9DsrPhqzCJFODasxc2QP3vhmA298m8J363fz2NVt6duhLv4aVyJFgnOOD3/czFOzVlPSjBeu7cCA\nTvX0eyp5opZFPqzdcYgHpySRuGU/v219Dk9e0566VcsVagwiudlx4BgPTkliwbo0ejSvydhBsfrd\nlFypG6qQZGU7/r3wJ56fs5aSJYxRV7TmJpUMEY845/hs+VYem7aSE1mOv1zZhiFdG6o1IaelZFHI\ntuxN56FPV/B9ym66NK7OMwNjaBpd0bN4JPLsPnycR6YmM3vlDuIaVeP5azvQuGYFr8OSIk7JwgPO\nOT5JSOXJmas4lpnNPZe04A8XNqW0SoZIiM1O3sHDU1dw6Fgm91/ektt6NKWkWrcSBA1we8DMuC6u\nAb1aRvPY9JWMnb2WmYnbGTsolvb1VDJECt6B9BP8bcZKpi7bSvt6lZl43bm0rFXJ67AkDKllEUKz\nk30lQ/YeyeAPFzblnktUMkQKzvx1aYyanMTuw8cZ8dvm3H1Rc7Vi5aypZVEE9G5fh/Ob1uSpz1fx\n1vwNzFm5g2cGxNC1qQq1Sd4dOZ7JU5+v5sPFm2lxTkXG/y6OmPpquUpo6WtIiFUpX5qxgzrw/m1d\nyczO5vq3F/n7llUyRM7e4o176P3KAib+uJlhPZsy4489lCikUChZFJIeLWoy556e3N6jCRN/3Mxl\nLy1g3uqdXoclxcSxE1k8OXMVg8cvooQZnww7n4f6tFG3phQaJYtCVD6qFI9c1ZYpwy+gUtlS3PZe\nPCMnLmPP4eNehyZFWOKW/Vz1j+955/tNDOnaiC/+dCFxjVX9WAqXxiw80LFhNWb+8ULe/HYDr32z\nnu/Wp/HY1e3od65Khsj/ZGRm89rX63n92w2cU6kME27rwoUtor0OSyKUzoby2Lqdhxg1JYllm/fT\nq1U0T/WPoZ7KMkS8NTsOct+kRFZuO8jATvV59Oq2VClX2uuwJAzporxiJCvb8d7Cn3huzlpKGIy6\nojVDujZSyZAIlJXteHvBRl76ah2Vy5Xi7/1juKxdba/DkjCmZFEMbdmbzl+mruC79bs5r3E1nh4Q\nS/NzVDIkUmzafYT7Ji1n6eb99ImpzZPXxFC9QpTXYUmYU7IoppxzTFm6lSdmruJoRhZ/uqQFd/RU\nyZBwlp3tmLDoZ57+YjVlSpVkTL92KnkvhUYX5RVTZsagzvXp2bImj09fxXNz1jIzaTtjB8bqfPow\nlLovnQcnJ7Fwwx56tYrm2YGx1Kpc1uuwRH5FLYsibnbyDv46LZm9RzK4/cIm3HtJS51bHwZOFp0c\nM2MVzjn+elVbrj+vgVoTUujUsggTvdvX5vymNfj756sZN38jc5J38PSAWM5vppIhxdWug8d46NMV\nzFuzi65NqvP8tR1oUL2812GJnJE6wouBKuVL8+ygWD64vSvZDm4Yv4iHPl3BQZUMKXZmJm3jspcX\n8H3Kbh69qi0T/9BNiUKKBXVDFTNHM7J48au1/PP7TURXKsOT18RwadtaXoclAew7ksFfpyUzM2k7\nHRpU5YVrO+hMNykSdDZUmEvcsp9RU5JYs+MQV8XW4W9921GzYhmvw5JczFu9k9GfrmB/egb3XNKS\nYT2bUkpnt0kRoTGLMNehQVWmj+jBW/M38NrXKb90a/TvWE+DpEXEoWMneGLmKibFp9K6diXe+30X\n2tat7HVYInmilkUYWO8vGbJ0835+0zKap/q3p3419YN7aWHKbh6YnMT2A0cZ3qsZIy9uQZlSOotN\nih51Q0WYrGzHhP/+xNg5awEY1bs1N3dTyZDCdjQji2dnr+HfC3+iac0KvHBdBzo2rOZ1WCKnpWQR\noXKWDOncqBrPDoyh+Tmak7kwJPy8j/s/SWTT7iP8vntjHry8NeWi1JqQoi3YZBHSUTYz621ma80s\nxcxGn2G7QWbmzCwux7pYM/uvma00sxVmpstag9Cgenn+c2sXXri2Aym7DtPnle/5x7z1nMjK9jq0\nsHU809eauPathWRkZvPhH7ry2NXtlCgkrIRsgNvMSgKvA5cCqcASM5vunFt1ynaVgJHA4hzrSgHv\nAzc75xLNrAagiwqCZGYM7Fyfni2j+duMlbzw1TpmrdjO2EGxxNav6nV4YSV56wHum5TI2p2HGHxe\nAx6+sg2VyqqUuISfULYsugApzrmNzrkM4COgXy7bPQGMBY7lWHcZkOScSwRwzu1xzmWFMNawFF2p\nDK/f2Im3b+7MvvQMrnn9B/7++WqOZuitzK/MrGxenbeea17/gX3pGfxr6Hk8MzBWiULCViiTRT1g\nS47lVP+6X5hZR6CBc27mKfu2BJyZzTGzpWb2YAjjDHuXtavNl/f+huvPa8DbCzbS+5UFLNyw2+uw\niq2UXYcY+OZCXvxqHX1i6vDlvT25qPU5XoclElKhTBa5nYbzy2i6mZUAXgLuy2W7UkAP4Cb/z/5m\ndvGvXsDsDjOLN7P4tLS0gok6TFUpV5qnB8Ty4R+6AnDj+MWMnpLEgaPq3QtWdrbjne820ufV79m8\nN53Xb+zEqzd0pGp5zTkh4S+UySIVaJBjuT6wLcdyJaA98K2Z/QR0A6b7B7lTgfnOud3OuXTgc6DT\nqS/gnHvbORfnnIuLjtbcxMG4oFlNZv+pJ8N6NmVS/BYufXE+X67c4XVYRd7mPekMHr+IJ2etpmeL\naL689zdcGVvH67BECk0ok8USoIWZNTGzKGAwMP3kg865A865ms65xs65xsAioK9zLh6YA8SaWXn/\nYPdvgFW/fgnJi3JRJXmoTxs+u7s71StEcceEBO7+cClph457HVqR45zjg8U/0/uVBazedpDnr+3A\n+N91JrqSSqtIZAnZ2VDOuUyw9wGqAAAL/UlEQVQzG4HvH39J4F3n3EozGwPEO+emn2HffWb2Ir6E\n44DPnXOzQhVrpIqtX5UZf+zBuPkbeHVeCt+v95UMGdBJJUMAdhw4xoNTkliwLo0ezWsydlAsdauW\n8zosEU/oojwBfIO2o6asIOHnfVzYoiZ/7x8TsaWznXN8tnwrj01byYksx1/6tOamrroaXsKTruCW\ns3ZyLuixs9fggAcub8Xvzm9MyQj6J7n78HEemZrM7JU7iGtUjeev7UDjmhW8DkskZJQsJM9S96Xz\n8NRk5q9Lo2PDqowdGEuLWuFfMmR28g4enrqCQ8cyue+yltx+YdOISpQSmYpEuQ8pnupXK8+/f38e\nL13fgU27j3Dlq9/zytz1ZGSGZ8mQA+knuPfj5dz5fgJ1qpZl5sgeDPtNMyUKkRw0n4Xkyszo37E+\nF7aI5vEZq3hp7jo+X7GdZwfFcm6D8CkZMn9dGqMmJ5F2+Dh/urgFI37bnNKamEjkV/RXIWdUs2IZ\n/nFDR8b/Lo79RzMY8MYPPDlzVbEvGXLkeCZ/mbqCW979kUplS/HZXd2599KWShQip6GWhQTl0ra1\n6Nq0Os98sYZ3vt/El6t28syAGC5oXtPr0M7a4o17uH9yIqn7jnJHz6b8+dKWlC2tCrEiZ6KvURK0\nymVL8/f+MXx0RzdKGNz4zmJGTS4+JUOOncjiyZmrGDx+EYYxadj5/KVPGyUKkSDobCjJk2Mnsnhp\n7jre+W4TNSpEMaZfe3q3r+11WKeVuGU/932SSMquw9zcrRGjr2hNhTJqWIvobCgJqbKlS/LQFW34\n7K7u1KhYhjvfT+CuDxLYdehY4J0LUUZmNi9+uZYBby7kyPFM/nNrF564pr0ShchZ0l+M5EtM/SpM\nH9Gdtxds5JV56/khZQ+PXNmGQZ3re14yZM2Og9w3KZGV2w4ysFN9Hr26LVXKab4JkbxQN5QUmA1p\nhxk9JYklP3lbMiQr2/H2go289NU6KpcrxVP9Y7i8XdHtIhPxkq7gFk9kZ/uqtD7zxRqyna9kyC0X\nFF7JkE27j3DfpOUs3byfK9rX5slr2lOjoirEipyOkoV4auv+ozw8dQXfrk3j3AZVGTsolpYhLBly\nsq7V01+sJqpkCZ64pj19O9T1vCtMpKhTshDPOeeYtnwbj89YyeHjmdx9UXPu6tWcqFIFe15F6r50\nHpycxMINe+jVKppnB8ZSq3LZAn0NkXAVbLLQALeEjJlxTcd6XNiiJo/PWMXLc9fzxYodPDMwho4N\nq+X7+Z1zfJKQypgZq3DO8cyAGK4/r4FaEyIhoFNnJeRqVCzDqzd05J+3xHHg6AkGvLmQJ2auIj0j\nM8/PuevgMW5/L54HJyfRrm5lZt/Tk8FdGipRiISIWhZSaC5uU4suTarz7Ow1/PP7TXy5agdP94+l\nR4uzKxkyM2kbj3yWzNGMLB69qi1DL2isiYlEQkwtCylUlcqW5slrYvj4jm6UKlGCIf9czAOfJHIg\nPXDJkH1HMhjx4VJGfLiMRjUqMGvkhdzao4kShUgh0AC3eObYiSxembeetxdspHqFKMb0bccVMXVy\n3Xbe6p2M/nQF+9MzuOeSlgzr2ZRSqhArkm8q9yFFXtnSJRnVuzXT7u5OdMUyDP9gKXdOSGDXwf+V\nDDl07AQPTk7ktvfiqVEhiml39+Dui5orUYgUMrUspEg4kZXN+O828vLc9ZQtVYJHrmxLvWrleHBy\nEtsPHGV4r2aMvLgFZUqpQqxIQdJ1FlIsbUg7zENTVvDjT3sBaFqzAi9c16FATrUVkV/TdRZSLDWL\nrshHd3Tj4/gtbN9/lOG9mlMuSq0JEa8pWUiRU6KEcUOXhl6HISI5aJRQREQCUrIQEZGAlCxERCQg\nJQsREQlIyUJERAJSshARkYCULEREJCAlCxERCShsyn2YWRrwcz6eoiawu4DC8VK4HAfoWIqqcDmW\ncDkOyN+xNHLORQfaKGySRX6ZWXww9VGKunA5DtCxFFXhcizhchxQOMeibigREQlIyUJERAJSsvif\nt70OoICEy3GAjqWoCpdjCZfjgEI4Fo1ZiIhIQGpZiIhIQBGbLMzsOTNbY2ZJZjbVzKqeZrveZrbW\nzFLMbHRhxxmImV1rZivNLNvMTns2hJn9ZGYrzGy5mRXJKQXP4liK9GcCYGbVzewrM1vv/5nrVH9m\nluX/TJab2fTCjvN0Ar3HZlbGzD72P77YzBoXfpTBCeJYhppZWo7P4XYv4gzEzN41s11mlnyax83M\nXvUfZ5KZdSrQAJxzEXkDLgNK+e8/CzybyzYlgQ1AUyAKSATaeh37KTG2AVoB3wJxZ9juJ6Cm1/Hm\n91iKw2fij3MsMNp/f3Ruv1/+xw57HWte3mPgLuAt//3BwMdex52PYxkKvOZ1rEEcS0+gE5B8msf7\nAF8ABnQDFhfk60dsy8I596VzLtO/uAion8tmXYAU59xG51wG8BHQr7BiDIZzbrVzbq3XcRSEII+l\nyH8mfv2A9/z33wOu8TCWsxXMe5zz+CYDF5uZFWKMwSouvy8BOecWAHvPsEk/4D/OZxFQ1czqFNTr\nR2yyOMWt+DLyqeoBW3Isp/rXFUcO+NLMEszsDq+DyYfi8pnUcs5tB/D/POc025U1s3gzW2RmRSWh\nBPMe/7KN/0vXAaBGoUR3doL9fRno77qZbGYNCie0AhfSv42wnoPbzOYCtXN56GHn3DT/Ng8DmcAH\nuT1FLusK/fSxYI4jCN2dc9vM7BzgKzNb4/+mUqgK4FiKxGcCZz6Ws3iahv7PpSnwtZmtcM5tKJgI\n8yyY97jIfA4BBBPnDGCic+64md2Jr8X025BHVvBC+pmEdbJwzl1ypsfN7BbgKuBi5+/0O0UqkPNb\nRn1gW8FFGJxAxxHkc2zz/9xlZlPxNc8LPVkUwLEUic8EznwsZrbTzOo457b7uwJ2neY5Tn4uG83s\nW6Ajvj52LwXzHp/cJtXMSgFVOHMXiVcCHotzbk+OxfH4xjCLo5D+bURsN5SZ9QZGAX2dc+mn2WwJ\n0MLMmphZFL6BvCJzxkqwzKyCmVU6eR/f4H6uZ1QUA8XlM5kO3OK/fwvwq1aTmVUzszL++zWB7sCq\nQovw9IJ5j3Me3yDg69N84fJawGM5pV+/L7C6EOMrSNOB3/nPiuoGHDjZFVogvB7h9+oGpODr31vu\nv508s6Mu8HmO7foA6/B923vY67hzOY7++L5RHAd2AnNOPQ58Z4Ik+m8ri+JxBHssxeEz8cdYA5gH\nrPf/rO5fHwe8479/AbDC/7msAG7zOu4zvcfAGHxfrgDKAp/4/45+BJp6HXM+juVp/99FIvAN0Nrr\nmE9zHBOB7cAJ/9/JbcCdwJ3+xw143X+cKzjD2ZF5uekKbhERCShiu6FERCR4ShYiIhKQkoWIiASk\nZCEiIgEpWYiISEBKFiJnwcwO53P/yf6rtTGzimY2zsw2+KvtLjCzrmYW5b8f1hfNSvGiZCFSSMys\nHVDSObfRv+odfFc9t3DOtcNX/bSm8xW8mwdc70mgIrlQshDJA/9Vss+ZWbJ/npDr/etLmNkb/pbC\nTDP73MwG+Xe7Cf+V3GbWDOgKPOKcywZfyQ/n3Cz/tp/5txcpEtTMFcmbAcC5QAegJrDEzBbgK9nR\nGIjBV2l2NfCuf5/u+K7CBWgHLHfOZZ3m+ZOB80ISuUgeqGUhkjc98FUqzXLO7QTm4/vn3gP4xDmX\n7Zzbga98xEl1gLRgntyfRDJO1vQS8ZqShUjenG6inzNNAHQUX00l8NUi6mBmZ/obLAMcy0NsIgVO\nyUIkbxYA15tZSTOLxjfl5Y/A9/gm0ilhZrWAXjn2WQ00B3C+OSvigcdPzjBnZi3MrJ//fg0gzTl3\norAOSORMlCxE8mYqkISvUunXwIP+bqcp+CqCJgPjgMX4ZpEDmMX/Tx6345s8KcXMVuCbS+Hk/AMX\nAZ+H9hBEgqeqsyIFzMwqOucO+1sHP+KbpXCHmZXDN4bR/QwD2yef41PgIRcm86tL8aezoUQK3kwz\nqwpEAU/4Wxw4546a2WP45kXefLqd/ZP0fKZEIUWJWhYiIhKQxixERCQgJQsREQlIyUJERAJSshAR\nkYCULEREJCAlCxERCej/AI5qKLfnCZGBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1107bd650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores = np.zeros((n_classes, n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "    scores[j][:] = np.mean(lrcv_L1.scores_[j], axis = 0)\n",
    "\n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs, 1))\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C is: [ 0.1   0.01  0.01]\n"
     ]
    }
   ],
   "source": [
    "print 'C is:',lrcv_L1.C_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -1.46951709e-01,   0.00000000e+00,  -6.86516911e-02,\n",
       "          0.00000000e+00,  -1.51963972e-01,   0.00000000e+00,\n",
       "          1.43890083e-01,  -8.69380428e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          5.21224058e-02,   0.00000000e+00,   2.59582069e-01,\n",
       "          0.00000000e+00,  -2.96419139e-02,  -2.26275362e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   5.42226189e-02,   0.00000000e+00,\n",
       "         -5.20020592e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.07891086e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.40089817e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   3.16064395e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   5.18553657e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.53904844e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   4.87822055e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.80820083e-01,   0.00000000e+00,\n",
       "         -1.50092136e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          2.42574368e-01,   0.00000000e+00,  -9.08067986e-02,\n",
       "          0.00000000e+00,  -4.36655731e-03,   7.70819369e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   2.28785042e-03,  -2.44927304e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   7.84328345e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -4.75171757e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          5.07926992e-03,   8.76953728e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -2.47716681e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.62735745e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,  -6.91152818e-05,\n",
       "         -1.80753206e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   1.04851006e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.70307652e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          8.57205498e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.84860922e-02,   0.00000000e+00,  -1.96565926e-02,\n",
       "          3.45140966e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -2.90957195e-02,   0.00000000e+00,  -1.08136831e-03,\n",
       "          0.00000000e+00,  -3.89727484e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -2.21733401e-03,  -3.54449182e-02,  -1.06583283e-01,\n",
       "          1.37969004e-02,   1.83544016e-02,   0.00000000e+00,\n",
       "          2.31952292e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -2.90842310e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -9.46802663e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   2.87805978e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -6.07720501e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.51283330e-02,   0.00000000e+00],\n",
       "       [  0.00000000e+00,   0.00000000e+00,   5.58656100e-03,\n",
       "          1.07594678e-02,  -4.81873561e-01,  -6.59313583e-01,\n",
       "          0.00000000e+00,  -2.42110982e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -6.61442591e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.33436913e-01,\n",
       "         -7.81412549e-02,  -1.51255307e-02,   1.00567469e-01,\n",
       "          0.00000000e+00,  -1.08555182e-02,  -2.64865791e-02,\n",
       "          4.98136827e-02,   6.93085206e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          2.09292943e-02,  -2.62981079e-02,   0.00000000e+00,\n",
       "          2.25701250e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -3.08516114e-02,   1.31038900e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -9.10324662e-02,   2.88297928e-02,  -8.65994957e-03,\n",
       "         -8.21510109e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -2.37929492e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -4.82675532e-05,\n",
       "          0.00000000e+00,   1.29421811e-02,   2.64034039e-02,\n",
       "          0.00000000e+00,  -1.41864947e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   5.20872427e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   2.72291948e-03,\n",
       "         -2.01316943e-02,   0.00000000e+00,  -1.20492708e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   2.37811673e-02,\n",
       "          1.80105320e-02,   0.00000000e+00,   1.15488463e-01,\n",
       "          1.75897620e-03,  -1.07602440e-01,   0.00000000e+00,\n",
       "         -1.01930478e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   2.63327545e-04,   0.00000000e+00,\n",
       "          3.17801639e-02,   0.00000000e+00,   7.66290320e-02,\n",
       "          2.32880702e-01,  -3.64089237e-02,   0.00000000e+00,\n",
       "          9.06690355e-02,   3.80942619e-02,   1.78838606e-01,\n",
       "         -2.59207680e-02,   2.05580656e-04,   0.00000000e+00,\n",
       "          0.00000000e+00,   9.85882668e-02,  -9.95249412e-02,\n",
       "         -3.89363662e-02,   0.00000000e+00,  -1.12469775e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.05159267e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -7.57377674e-03,   2.03601675e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.55548286e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.56507678e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -4.28754816e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -3.92535305e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.07242059e-03,   6.82553253e-02,\n",
       "          9.53410337e-02,   0.00000000e+00,   1.77381464e-02,\n",
       "          9.90350057e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          3.54705829e-02,   1.07902046e-02,   5.38219979e-03,\n",
       "         -3.68787220e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.35788324e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -6.67758756e-02,   7.19562192e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   5.58941977e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,  -8.24233118e-02,\n",
       "         -1.59929485e-03,   6.69602929e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   5.89215007e-02,   0.00000000e+00,\n",
       "         -3.80471217e-03,  -6.89630164e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.25413719e-02,   0.00000000e+00,\n",
       "         -5.67209831e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.48528804e-02,\n",
       "          3.55977480e-03,   0.00000000e+00,   2.33194883e-02,\n",
       "          0.00000000e+00,   1.40930764e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.02510656e-03,\n",
       "         -4.76066140e-07,   2.30073442e-03,  -4.68641977e-02,\n",
       "          0.00000000e+00,   3.21784494e-02,   0.00000000e+00,\n",
       "         -5.31151084e-07,   7.99778633e-02,   2.95463216e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -6.50874045e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00],\n",
       "       [  0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -3.60570037e-02,   8.30441913e-01,   3.65960176e-01,\n",
       "         -9.22812201e-01,   4.38734334e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   5.94619811e-02,\n",
       "         -3.27515699e-02,   0.00000000e+00,  -2.64475539e-01,\n",
       "          5.29795081e-02,   4.68775141e-02,  -5.56948593e-02,\n",
       "          0.00000000e+00,   4.71877970e-02,   0.00000000e+00,\n",
       "         -9.34790984e-02,  -9.97504551e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -6.89102572e-03,   3.91139326e-02,   0.00000000e+00,\n",
       "         -5.76433092e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -8.13779449e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.04811277e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.93261874e-02,\n",
       "          2.75007137e-03,  -2.80337470e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   2.07687974e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.03970320e-02,\n",
       "          0.00000000e+00,  -1.96824970e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   1.51522098e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -4.57539625e-03,  -4.57395056e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          2.49986572e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.59553003e-02,   0.00000000e+00,\n",
       "         -4.46261529e-03,   0.00000000e+00,  -8.93164971e-02,\n",
       "          0.00000000e+00,   2.22516043e-01,   0.00000000e+00,\n",
       "          1.85251771e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.35272037e-02,   0.00000000e+00,\n",
       "         -5.55804088e-02,   0.00000000e+00,  -4.71021339e-02,\n",
       "         -3.32529509e-01,   5.47116999e-02,   3.74280537e-02,\n",
       "         -7.52645861e-02,  -7.38021708e-03,  -2.07844435e-01,\n",
       "          3.01004638e-04,  -2.01948475e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -9.93996767e-02,   1.36400421e-01,\n",
       "          6.84969112e-02,  -3.74529865e-02,   2.52515808e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.06882908e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -5.85367838e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.02558080e-02,\n",
       "          1.50761206e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   7.59898564e-04,   0.00000000e+00,\n",
       "         -9.27131574e-03,  -3.82397735e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   7.52908110e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   3.49625450e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.87100227e-02,\n",
       "         -1.37486664e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.14417198e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.08864472e-05,   0.00000000e+00,  -5.97226733e-05,\n",
       "          5.73771624e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          2.31608801e-02,  -7.99695616e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          5.56132298e-03,   0.00000000e+00,   2.71309288e-02,\n",
       "         -5.64818643e-02,  -6.09802654e-02,   0.00000000e+00,\n",
       "         -1.62640954e-03,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.72299093e-04,   1.46612896e-01,   0.00000000e+00,\n",
       "         -3.35636301e-02,   3.64708750e-02,   0.00000000e+00,\n",
       "          3.45064826e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -3.41696385e-03,\n",
       "          1.42406329e-02,   0.00000000e+00,   2.31466828e-02,\n",
       "         -3.87935587e-02,   0.00000000e+00,   3.52369581e-02,\n",
       "         -8.04564131e-03,  -4.04749343e-02,   4.29292931e-02,\n",
       "          0.00000000e+00,  -3.61886843e-03,   0.00000000e+00,\n",
       "          4.50737170e-04,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -5.01007278e-03,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   9.77216001e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression + L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[0.001, 0.05, 0.01, 0.1, 1], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "Cs = [0.01, 0.1, 1, 10]\n",
    "Cs = [0.0001, 0.001, 0.01, 0.1, 1, 10]\n",
    "Cs = [0.001, 0.05, 0.01, 0.1, 1]\n",
    "lrcv_L2 = LogisticRegressionCV(Cs=Cs, cv=5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L2.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.49587921, -0.26307599, -0.29733283, -0.26037237, -0.26753789],\n",
       "        [-0.50947181, -0.30505979, -0.3361879 , -0.29941736, -0.2893818 ],\n",
       "        [-0.49902898, -0.28635187, -0.30978232, -0.28829358, -0.30260933],\n",
       "        [-0.51730404, -0.39405415, -0.3783421 , -0.39584412, -0.39979993],\n",
       "        [-0.50085463, -0.28761966, -0.31334583, -0.28737838, -0.29465703]]),\n",
       " 1: array([[-0.59380283, -0.51902531, -0.51634307, -0.52724791, -0.56323838],\n",
       "        [-0.6078046 , -0.56168831, -0.55264686, -0.57320876, -0.65119695],\n",
       "        [-0.58432796, -0.48918847, -0.49436192, -0.49486751, -0.53179659],\n",
       "        [-0.60843998, -0.53601254, -0.53546485, -0.54285772, -0.56903184],\n",
       "        [-0.59874009, -0.50837292, -0.51621492, -0.51118276, -0.53452217]]),\n",
       " 2: array([[-0.62477446, -0.57743615, -0.57545941, -0.5810174 , -0.61067622],\n",
       "        [-0.62775192, -0.61776281, -0.59676505, -0.63071766, -0.69580488],\n",
       "        [-0.6122289 , -0.5684452 , -0.56045504, -0.57498952, -0.61901489],\n",
       "        [-0.64985256, -0.67076893, -0.65766835, -0.68569085, -0.75009715],\n",
       "        [-0.62505436, -0.58590322, -0.58073993, -0.58971986, -0.61666243]])}"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd4VGX6//H3nZ4AKfSS0EF6TWAF\nBbvYYO0oIgmgri5frLt2XdtPXXVde2MJoKtiY8VFRbFg1wTpHVFMqAFCCklIu39/zODGkGQGyOTM\nTO7Xdc3FzJnnTD4nQ+aec87zPEdUFWOMMaYuIU4HMMYY4/+sWBhjjPHIioUxxhiPrFgYY4zxyIqF\nMcYYj6xYGGOM8ciKhTHGGI+sWBhjjPHIioUxxhiPwpwOUF9atmypnTt3djqGMcYElCVLluxW1Vae\n2vm0WIjIGOAJIBSYoaoPVXs+FXgE2Ope9LSqznA/93fgLFx7Px8D12odc5N07tyZzMzMet8GY4wJ\nZiKyxZt2PisWIhIKPAOcCmQDGSIyX1XXVGs6V1WnVVt3BDASGOBe9BUwGvjcV3mNMcbUzpfnLIYB\nm1R1s6qWAq8D47xcV4EoIAKIBMKBnT5JaYwxxiNfFosOQFaVx9nuZdWdLyIrROQtEUkCUNVvgc+A\n7e7bQlVdW31FEblSRDJFJDMnJ6f+t8AYYwzg22IhNSyrfs7hPaCzqg4AFgGzAUSkO9AbSMRVYE4S\nkVGHvJjqi6qarKrJrVp5PD9jjDHmCPmyWGQDSVUeJwLbqjZQ1T2qesD98CVgqPv+ucB3qlqoqoXA\nB8AffJjVGGNMHXxZLDKAHiLSRUQigPHA/KoNRKRdlYdjgYOHmn4FRotImIiE4zq5fchhKGOMMQ3D\nZ72hVLVcRKYBC3F1nZ2pqqtF5F4gU1XnA9NFZCxQDuwFUt2rvwWcBKzEdejqQ1V9z1dZjTHG1E2C\n5bKqycnJeiTjLMoqKnlk4XouP7YTiQkxPkhmjDH+S0SWqGqyp3aNfrqPrbnFvPb9r6SlZ5BXXOZ0\nHGOM8UuNvlh0btmEFyYO5Zc9+/nTy0soLa90OpIxxvidRl8sAEZ0b8lD5w3g2817uOXtFQTLoTlj\njKkvQTOR4NE6f2gi2bnFPL5oA4nNY7jh1J5ORzLGGL9hxaKK6Sd3Jyu3iCc/2UhifDQXpSR5XskY\nYxoBKxZViAgPntefnfkl3DZvJe3iozi+h40MN8YYO2dRTXhoCM9OGEL31k25+pUfWbs93+lIxhjj\nOCsWNWgWFU56WgpNIkNJS89ge16x05GMMcZRVixq0S4umpmpKRSUlJGWnkFBiY3BMMY0XlYs6tC3\nfRzPXjaUjbsKuebfP1JWYWMwjDGNkxULD0b3bMUDf+zHlxt3c8e8VTYGwxjTKFlvKC+MH9aR7Nxi\nnv5sE0nNo5l2Ug+nIxljTIOyYuGlG0/rSXZuEY9+tIEOCdGcOzjR6UjGGNNgrFh4SUR4+IIB7Mgv\n4a9vraBNbBQjurV0OpYxxjQIO2dxGCLDQnnhsmQ6tWjCVS8vYePOAqcjGWNMg7BicZjiYsJJT00h\nMiyU1PQMdhWUOB3JGGN8zorFEUhqHsPM1GT27i9l8qwM9h8odzqSMcb4lBWLIzQgMZ6nLx3Mmm35\n/N9rSym3MRjGmCBmxeIonNy7DfeM7cun63bxt/dW2xgMY0zQst5QR2nisZ3Jzi3mhS82k5QQw1Wj\nuzkdyRhj6p0Vi3pw85heZO8r5sEP1tEhIZqzB7R3OpIxxtQrKxb1ICREeOzCgezMK+GGN5bTJjaK\nlM7NnY5ljDH1xs5Z1JOo8FBeujyZDvHRXDEnk59yCp2OZIwx9caKRT1KaBLBrLQUQkVIS89gd+EB\npyMZY0y9sGJRzzq1aMJLk5LZmV/C1NmZFJdWOB3JGGOOmhULHxjSMYEnxg9mefY+rpu7lIpK61Jr\njAlsVix8ZEy/ttxxVh8Wrt7J/QvWOB3HGGOOivWG8qEpx3UhO7eI9K9/ISkhhsnHdXE6kjHGHBEr\nFj52x1l92LavmPsWrKF9fDRj+rV1OpIxxhw2OwzlY6Ehwj8vHszAxHiufX0pP/6a63QkY4w5bFYs\nGkB0RCgzJiXTJjaKqbMz2bJnv9ORjDHmsPi0WIjIGBFZLyKbROSWGp5PFZEcEVnmvk2t8lxHEflI\nRNaKyBoR6ezLrL7Wsmkk6WkpVKqSmp5B7v5SpyMZY4zXfFYsRCQUeAY4A+gDXCIifWpoOldVB7lv\nM6osnwM8oqq9gWHALl9lbSjdWjXlpcuT2bqvmCvmZFJSZmMwjDGBwZd7FsOATaq6WVVLgdeBcd6s\n6C4qYar6MYCqFqpqke+iNpyUzs157MKBZG7J5cY3l1NpYzCMMQHAl8WiA5BV5XG2e1l154vIChF5\nS0SS3Mt6AvtE5B0RWSoij7j3VH5HRK4UkUwRyczJyan/LfCRcwa255YzerFgxXYeXrjO6TjGGOOR\nL4uF1LCs+tfo94DOqjoAWATMdi8PA44HbgJSgK5A6iEvpvqiqiaranKrVq3qK3eDuGpUVyYM78gL\nizfz8ndbnI5jjDF18mWxyAaSqjxOBLZVbaCqe1T14Gx7LwFDq6y71H0Iqxz4DzDEh1kbnIhwz9i+\nnNSrNXe/u4pP1u50OpIxxtTKl8UiA+ghIl1EJAIYD8yv2kBE2lV5OBZYW2XdBBE5uLtwEhB0c2aE\nhYbw1CWD6dM+lmmvLmVldp7TkYwxpkY+KxbuPYJpwEJcReANVV0tIveKyFh3s+kislpElgPTcR9q\nUtUKXIegPhGRlbgOab3kq6xOahIZxsxJKTRvEsHk2Rlk7Q2K8/jGmCAjqsHRGyc5OVkzMzOdjnHE\nNuws4PznvqFNbBRv/2kEcTHhTkcyxjQCIrJEVZM9tbMR3H6iZ5tmvDBxKFv27OeqVzI5UG5jMIwx\n/sOKhR8Z0a0lf79gAN9t3sstb68kWPb6jDGBz2ad9TPnDk4ke28xj328gcSEaG487RinIxljjBUL\nfzTtpO5k5xbz1KebSEyI5uKUjk5HMsY0clYs/JCIcP+5/diWV8xt81bRNi6a0T0Da9ChMSa42DkL\nPxUeGsKzE4bQo3VTrnllCWu25TsdyRjTiFmx8GPNosJJT0uhWVQ4k2dlsD2v2OlIxphGyoqFn2sX\nF016WgqFB8pJS88gv6TM6UjGmEbIikUA6N0ulmcnDGHjrkL+/O8fKauodDqSMaaRsWIRIEb1bMWD\n5/bny427ue0dG4NhjGlY1hsqgFyUkkR2bhFPfrqJpOYxTD+5h9ORjDGNhBWLAHP9qT3Jzi3mHx9v\noEN8NOcPTXQ6kjGmEbBiEWBEhIfOH8D2vBJufnsF7eKiGNG9pdOxjDFBzs5ZBKCIsBCenziULi2b\ncNUrS9iws8DpSMaYIGfFIkDFRbvGYESFh5KWnsGu/BKnIxljgpgViwCWmBBDemoKuUWlTJ6dwf4D\n5U5HMsYEKSsWAa5fhzieuXQIa7blM+3VHym3MRjGGB+wYhEETuzVmnvH9eOz9TncNX+1jcEwxtQ7\n6w0VJC77Qyeyc4t5fvFPJCXEcPUJ3ZyOZIwJIlYsgshfTz+GrfuKefjDdXRIiGbswPZORzLGBAkr\nFkEkJER45IIB7Mgr5qY3ltM2NophXZo7HcsYEwTsnEWQiQoP5cWJySQmRHPFnEw27Sp0OpIxJghY\nsQhCCU0imJU2jLAQIW3WD+QUHHA6kjEmwFmxCFIdW8Twr9QUcgoOMHVOJsWlFU5HMsYEMCsWQWxQ\nUjxPjh/Miux9TH99KRWV1qXWGHNkrFgEudP6tuWus/vw8Zqd3PffNTYGwxhzRKw3VCOQNrILWXuL\nmfn1zyQmRDP1+K5ORzLGBBgrFo3E7Wf1Ztu+Yh54fy0d4qM5o387pyMZYwKIHYZqJEJDhH+OH8Sg\npHium7uMJVtynY5kjAkgViwakajwUGZcnkzbuCiumJPJL7v3Ox3JGBMgrFg0Mi2aRpKemkKlKqnp\nP7B3f6nTkYwxAcCnxUJExojIehHZJCK31PB8qojkiMgy921qtedjRWSriDzty5yNTddWTZlxeTLb\n8kq4Yk4mJWU2BsMYUzefFQsRCQWeAc4A+gCXiEifGprOVdVB7tuMas/dByz2VcbGLLlzcx6/aBBL\ntuRy4xvLqbQxGMaYOnhVLERkpIg0cd+/TET+ISKdPKw2DNikqptVtRR4HRjnbTARGQq0AT7ydh1z\neM4a0I7bzuzFgpXbeejDdU7HMcb4MW/3LJ4DikRkIPBXYAswx8M6HYCsKo+z3cuqO19EVojIWyKS\nBCAiIcBjwF/q+gEicqWIZIpIZk5OjpebYqq64viuTPxDJ178YjNzvv3F6TjGGD/lbbEoV9fQ33HA\nE6r6BNDMwzpSw7LqxzreAzqr6gBgETDbvfwa4H1VzaIOqvqiqiaranKrVq08boQ5lIhw9zl9OKV3\na/42fzWL1ux0OpIxxg95WywKRORW4DJggft8RLiHdbKBpCqPE4FtVRuo6h5VPTgl6kvAUPf9Y4Fp\nIvIL8ChwuYg85GVWc5jCQkN48pLB9OsQx/+9tpQV2fucjmSM8TPeFouLgQPAFFXdgetw0iMe1skA\neohIFxGJAMYD86s2EJGqw4jHAmsBVHWCqnZU1c7ATcAcVT2kN5WpPzERYcyYlEzzJhFMnpVJ1t4i\npyMZY/yI13sWuA4/fSkiPYFBwGt1raCq5cA0YCGuIvCGqq4WkXtFZKy72XQRWS0iy4HpQOqRbISp\nH62bRTF7cgql5RWkpv9AXlGZ05GMMX5CvJmFVESWAMcDCcB3QCZQpKoTfBvPe8nJyZqZmel0jKDw\n3eY9TPzX9wzpmMCcKcOIDAt1OpIxxkdEZImqJntq5+2ehahqEXAe8JSqngv0PZqAxn/9oWsLHr1w\nIN//vJe/vrXCxmAYY7yedVZE5FhgAjDFvcy+bgaxcYM6kJ1bzCML15OYEM1fTu/ldCRjjIO8LRbX\nAbcC89znHboCn/kulvEH15zQjay9RTzz2U8kJsRwybCOTkcyxjjEq2KhqouBxSLSTESaqupmXCek\nTRATEe77Yz+25ZVwx39W0TYuihOPae10LGOMA7yd7qO/iCwFVgFrRGSJiNg5i0YgPDSEZycM4Zg2\nzZj27x9ZvS3P6UjGGAd4e4L7BeAGVe2kqh2BG3ENojONQNPIMNLTUoiNDmfyrAy27St2OpIxpoF5\nWyyaqOpv5yhU9XOgiU8SGb/UJjaK9LQUig5UkJaeQX6JjcEwpjHxtlhsFpE7RaSz+3YH8LMvgxn/\n06ttLM9dNpSfcgq55pUfKS2vdDqSMaaBeFssJgOtgHeAee77ab4KZfzXcT1a8uB5/flq025ufWcl\n3gzqNMYEPm97Q+VivZ+M24XJSWTnFvPEJxtJah7Ndaf0dDqSMcbH6iwWIvIeh04r/htVHVvbcya4\nXXdKD7Jzi/nnoo0kJsRwwdBEpyMZY3zI057Fow2SwgQcEeHB8/qzPa+YW95eQbu4KEZ2b+l0LGOM\nj3g1kWAgsIkEnZFXXMaFz3/D9n0lvHn1sfRqG+t0JGPMYajXiQRFZKX70qdVb1+KyOMi0uLo45pA\nFRcdTnraMKIjQpmcnsHO/BKnIxljfMDb3lAfAAtwTSQ4AdflUL8EdgCzfJLMBIwO8dHMTE0hr7iM\ntPQMCg+UOx3JGFPPvC0WI1X1VlVd6b7dDoxW1YeBzr6LZwJFvw5xPD1hCOt3FvDnf/9IeYWNwTAm\nmHhbLJqKyPCDD0RkGNDU/dC+RhoATjymNfeN68fiDTnc+e4qG4NhTBDxdoryqcBMEWkKCJAPTBGR\nJsCDvgpnAs+lwzuSnVvEs5+7pjX/84ndnY5kjKkH3g7KywD6i0gcrh5U+6o8/YZPkpmAddNpx/zu\nwknjBnVwOpIx5ih5VSzcReJuYJT78WLgXlW1+arNIUJChEcuHMCO/BL+8uYK2sZGMbyrdZozJpB5\ne85iJlAAXOS+5QPpvgplAl9kWCgvThxKYvNorpiTyaZdBU5HMsYcBW+LRTdVvVtVN7tv9wBdfRnM\nBL74mAhmpw0jIiyE1PQMcgoOOB3JGHOEvC0WxSJy3MEHIjISsCvgGI+Smsfwr0kp7C48wJTZGRSV\nWuc5YwKRt8XiauAZEflFRLYATwN/8l0sE0wGJsXz1CVDWLU1j+mvLaWi0rrUGhNovCoWqrpMVQcC\nA4D+qjpYVZf7NpoJJqf2acPd5/Rl0dpd3PPeahuDYUyA8TRF+Q21LAdAVf/hg0wmSE0a0ZmsvUXM\n+OpnkhJiuGKUnfYyJlB46jrbrEFSmEbjtjN7s3VfMQ+8v5YOCdGc2b+d05GMMV6os1i4ez0ZU29C\nQoTHLx7EzvzvuG7uMtrERjK0U3OnYxljPPD2BPdvRORHXwQxjUdUeCgzJqXQPi6KqbMz+Xn3fqcj\nGWM8OOxigWtuKGOOSvMmEcxKGwZAWvoP7Cm0MRjG+LMjKRYLvG0oImNEZL2IbBKRW2p4PlVEckRk\nmfs21b18kIh8KyKr3RdauvgIcho/17llE2ZMSmF7XglT52RSUlbhdCRjTC0Ou1io6h3etBORUOAZ\n4AygD3CJiPSpoelcVR3kvs1wLysCLlfVvsAY4J8iEn+4WY3/G9opgX9ePIhlWfu4fu4yKm0MhjF+\nydvLqhaISH61W5aIzBOR2vo/DgM2uacHKQVeB8Z58/NUdYOqbnTf3wbsAlp5s64JPGf0b8ftZ/bm\ng1U7+H/vr3U6jjGmBt5ez+IfwDbgVVznLMYDbYH1uCYZPKGGdToAWVUeZwPDa2h3voiMAjYA16tq\n1XUOXmgpAvjJy6wmAE05rstvYzASE6JJHdnF6UjGmCq8PQw1RlVfUNUCVc1X1ReBM1V1LpBQyzo1\nnQivfozhPaCzqg4AFgGzf/cCIu2Al4E0VT3kOp0icqWIZIpIZk5OjpebYvyRiHDXOX05pXcb7vnv\nGj5avcPpSMaYKrwtFpUicpGIhLhvF1V5rraDzNlAUpXHibj2Tv63ouoeVT3YDeYlYOjB50QkFtfJ\n9DtU9buafoCqvqiqyaqa3KqVHaUKdKEhwpOXDGJAhzimv76UZVn7PK9kjGkQ3haLCcBEXOcOdrrv\nXyYi0cC0WtbJAHqISBcRicB16Gp+1QbuPYeDxgJr3csjgHnAHFV908uMJgjERIQxY1IKrZpFMnV2\nBll7i5yOZIzB+4kEN6vqOaraUlVbue9vUtViVf2qlnXKcRWShbiKwBuqulpE7hWRse5m093dY5cD\n04FU9/KLcF2VL7VKt9pBR7GdJoC0ahZJeuowyiqUSek/sK+o1OlIxjR64s3snyLSE3gOaKOq/URk\nADBWVe/3dUBvJScna2ZmptMxTD36fvMeJv7rBwZ1jOflKcOIDAt1OpIxQUdElqhqsqd23h6Gegm4\nFSgDUNUVuA4rGeMzw7u24JELB/DDz3u56c0VNgbDGAd523U2RlV/ODg1uZtd8sz43LhBHdi6r5i/\nf7iexIRobh7Ty+lIxjRK3haL3SLSDXfPJxG5ANjus1TGVHH16G5k7S3muc9/IikhhkuHd3Q6kjGN\njrfF4s/Ai0AvEdkK/Iyrh5QxPici3DeuL9vzirnz3VW0i4vixF6tnY5lTKPi7TmLrUA68ACuaTs+\nBib5KpQx1YWFhvD0pUPo1bYZf371R1ZtzXM6kjGNirfF4l3gHFwnuLcBhYBdhMA0qKaRYcxMTSE+\nOpy0WRls3VfsdCRjGg1vu86uUtV+DZDniFnX2cZj/Y4CLnjuG9rFR/Hmn0YQFx3udCRjAlZ9d539\nRkT6H2UmY+rFMW2b8fzEoWzO2c/VryyhtPyQacOMaVS8+dJ/tLwtFscBS9wXMlohIitFZIUvgxlT\nl5HdW/LQ+QP45qc93PLOigb5YzHG3+wrKuXO/6zi1ndW+vxnedsb6gyfpjDmCFwwNJGtucU8vmgD\niQkx3HBqT6cjGdMgKiuVuZlZ/P3DdeQVlzFpRGdUlWpj4eqVV8VCVbf4LIExR2H6yd3Jyi3iyU82\nkpgQzUXJSZ5XMiaALcvax93vrmJ5dh7DOjfnnnF96d0u1uc/19s9C2P8kojw4Hn92ZFXwm3vrKR9\nXDTH9WjpdCxj6t2ewgP8/cP1zM3MonWzSJ4YP4ixA9v7dG+iqsO+Brcx/iY8NIRnLxtC99ZN+dMr\nS1i7Pd/pSMbUm/KKSuZ8+wsnPvo5b/+YzZWjuvLJjaMZN6hDgxUKsGJhgkRsVDgzU1NoEhlKWnoG\nO/JKnI5kzFHL+GUv5zz9NXe9u5r+iXF8eN3x3HZmb5pFNXx3cSsWJmi0j49mZmoKBSVlpM3KoKCk\nzOlIxhyRXfklXD93GRc+/y15RaU8O2EIr0wZTvfWzRzLZMXCBJW+7eN49rKhbNhZwJ9fXUpZhY3B\nMIGjrKKSGV9u5qTHFrNgxXamndidRTeO5sz+7Rr0kFNNrFiYoDO6Zyse+GM/vtiQw53/WWVjMExA\n+GbTbs544kvuX7CWlM4JfHT9KG46/RhiIvyjH5J/pDCmno0f1pHs3GKe/mwTiQnRTDuph9ORjKnR\ntn3FPPD+Whas2E5S82hmXJ7Myb1bO74nUZ0VCxO0bjytJ9m5RTz6kWvQ3h8Hd3A6kjG/OVBewYwv\nf+bpTzdRqcoNp/bkylFdiQr3z8sHW7EwQUtEePiCAezIL+Evby2nTWwUx3Zr4XQsY/h8/S7ueW8N\nP+/ez+l923DHWX1Iah7jdKw62TkLE9Qiw0J54bJkOrVowlUvZ7JpV4HTkUwjlrW3iCvmZJKangHA\n7MnDeGFist8XCrBiYRqBuJhw0lNTiAgLZdLMDHYV2BgM07BKyip4/OMNnPKPxXy9aTc3j+nFh9cd\nz+ierZyO5jUrFqZRSGoew8zUZPbuL2XKrEz2Hyh3OpJpBFSVj1bv4JR/LOaJTzZyap82fHLjaK4+\noRuRYf55bqI2VixMozEgMZ6nLx3M6m15TH9tKeU2BsP40M+795M2K4MrX15CTEQor14xnKcvHUK7\nuGinox0RKxamUTm5dxvuGduXT9a5TjDaGAxT34pKy/n7h+s4/fEvWPJLLnee3YcF049nRLfAnuDS\nekOZRmfisZ3Jyi3mxS82k9Q8mitHdXM6kgkCqsr7K3dw/4I1bM8r4bwhHbjljF60bhbldLR6YcXC\nNEq3jOnF1txi/t/762gfH83ZA9o7HckEsI07C7h7/mq++WkPfdrF8tQlg0nu3NzpWPXKioVplEJC\nhMcuGsjO/BJueGM5bWOjgu6P2/heQUkZTyzayKxvfiEmIpT7xvXl0uGdCA3xr9HX9cHOWZhGKyo8\nlJcuT6ZDfDRT52SyOafQ6UgmQKgq85Zmc9Jji/nX1z9zYXIin910AhOP7RyUhQKsWJhGLqFJBLPS\nUggRITU9gz2FB5yOZPzcmm35XPTCt1w/dznt46KYd81IHjxvAC2aRjodzaesWJhGr1OLJsyYlMzO\n/BKmzM6kuLTC6UjGD+UVlXH3u6s4+6kv+SlnPw+f359514xkUFK809EahE+LhYiMEZH1IrJJRG6p\n4flUEckRkWXu29Qqz00SkY3u2yRf5jRmSMcEnhg/iOXZ+7hu7lIqKq1LrXGprFTmZvzKiY99zsvf\nbWHiHzrx2Y0ncHFKR0KC9JBTTXx2gltEQoFngFOBbCBDROar6ppqTeeq6rRq6zYH7gaSAQWWuNfN\n9VVeY8b0a8cdZ/Xhvv+u4YEFa7nrnD5ORzIOW5G9jzvfXc3yrH0kd0rgnnHD6Ns+zulYjvBlb6hh\nwCZV3QwgIq8D44DqxaImpwMfq+pe97ofA2OA13yU1RgAphzXhay9Rcz8+meSmkeTNrKL05GMA/bu\nL+WRhet4PSOLFk0i+cdFAzl3cAe/u8ZEQ/JlsegAZFV5nA0Mr6Hd+SIyCtgAXK+qWbWsaxcjMA3i\nzrP7sG1fMff+dw3t46M5vW9bpyOZBlJRqbz6w688unA9hQfKmTKyC9ee0oNmUeFOR3OcL89Z1FSC\nqx8Ifg/orKoDgEXA7MNYFxG5UkQyRSQzJyfnqMIac1BoiPDE+MEMSIxn+mtLWfqrHf1sDJZs2cs5\nT33Fnf9ZRZ92sXxw7fHccXYfKxRuviwW2UBSlceJwLaqDVR1j6oe7Kv4EjDU23Xd67+oqsmqmtyq\nVeBM9Wv8X3REKP+alEyb2Cimzs5ky579TkcyPrKroIQb3ljG+c99S25RKU9fOphXrxhOzzbNnI7m\nV3xZLDKAHiLSRUQigPHA/KoNRKRdlYdjgbXu+wuB00QkQUQSgNPcy4xpMC2bRpKelkKFKmnpGeTu\nL3U6kqlHZRWV/Ournzn50cW8t3wb15zQjUU3jObsAe0b9bmJ2vjsnIWqlovINFwf8qHATFVdLSL3\nApmqOh+YLiJjgXJgL5DqXneviNyHq+AA3HvwZLcxDalbq6a8ODGZy2Z8z5UvZ/LylOF+e41k471v\nf9rD3fNXsWFnIaN6tuJv5/Sha6umTsfyaxIsUzQnJydrZmam0zFMkHpv+Tb+77WlnDWgHU+NH9yo\n+tcHk+15rskj31u+jcSEaO46uw+n9mnTqPckRGSJqiZ7amcTCRrjhXMGtmfrvmIe+mAdiQnR3HpG\nb6cjmcNQWu465PTUpxspr1SuPbkHV5/QzfYSD4MVC2O8dNWormTtLeKFxZtJSojhsj90cjqS8cIX\nG3L42/zVbN69n1N6t+Gus/vQsUWM07ECjhULY7wkItwzti/b80q4691VtI+P4qRebZyOZWqRtbeI\n+xesYeHqnXRuEUN6agon9mrtdKyAZRMJGnMYwkJDeOqSwfRpH8u0V5eyMjvP6UimmpKyCp5YtJFT\n/rGYLzbs5i+nH8PC60dZoThKViyMOUxNIsOYOSmFhJgIJs/OIDu3yOlIxm3Rmp2c9vgXPL5oA6f0\nacMnN47mzyd2JzLMzk0cLSsWxhyB1rFRpKelUFJWQVp6BnnFZU5HatR+2b2fybMymDonk4iwEP49\ndTjPXDqE9vHRTkcLGlYsjDlCPds044WJQ/llz37+9PISSssrnY7U6BSVlvPowvWc9vgXfL95D7ef\n2ZsPrj2ekd1bOh0t6FixMOZ+0BQPAAAQGElEQVQojOjWkofPH8C3m/dw89srCJZxS/5OVXl/5XZO\neWwxT3+2ibMGtOOzm07gilFdCQ+1jzVfsN5Qxhyl84YksjW3mMc+3kBiQjQ3nnaM05GC2qZdBfxt\n/hq+2rSbXm2b8c/xgxnWpbnTsYKeFQtj6sG0k7qTnVvMU59uIikhhotSkjyvZA5L4YFynvxkIzO/\n+pnoiFDuGduXCcM7EmZ7Eg3CioUx9UBEuP/cfmzLK+bWeStpGxfFqJ42E3J9UFXmL9/GAwvWsqvg\nABcnJ/GXMcfQsmmk09EaFSvJxtST8NAQnp0whB6tm3LNv39kzbZ8pyMFvLXb87n4he+49vVltImN\nYt41I3j4ggFWKBxgxcKYetQsKpz0tBSaRoYxeVYG2/OKnY4UkPKKy/jb/NWc/dRXbNxVwIPn9ec/\nfx7J4I4JTkdrtKxYGFPP2sVFk56WQuGBctLSMygosTEY3qqsVN7IzOKkRz9n9re/cMmwJD676QQu\nGdaRUJvp11FWLIzxgd7tYnl2whA27irkmn//SFmFjcHwZGV2Huc//w1/fWsFnVrE8N6047j/j/2J\nj4lwOprBioUxPjOqZysePLc/X27cze3zVtoYjFrk7i/ltnkrGfvMV2TtLeLRCwfy1p9G0K9DnNPR\nTBXWG8oYH7ooJYms3KLfutT+38k9nI7kNyoqldd++JVHP1pPQUk5aSO6cN2pPYiNCnc6mqmBFQtj\nfOyGU3v+NmivQ0I05w1JdDqS45ZsyeXu+atYtTWf4V2ac8+4vvRqG+t0LFMHKxbG+JiI8ND5A9ie\nV8LNb6+gbWwUIxrp3EU5BQd4+MN1vLUkmzaxkTx5yWDOGdCuUV/WNFDYOQtjGkBEWAjPTxxK5xZN\nuOqVJWzYWeB0pAZVXlFJ+tc/c9Jjn/Pusq1cNborn954AmMHtrdCESCsWBjTQOKiXWMwosJDSUvP\nYFd+idORGsR3m/dw9lNfcc97axiUFM8H147i1jN60yTSDmwEEisWxjSgxIQYZk5KIbeolMmzM9h/\noNzpSD6zM7+E6a8tZfyL31FQUs7zlw1lzuRhdG/d1Olo5ghYsTCmgfVPjOOZS4ewZls+0179kfIg\nG4NRWl7JC4t/4qRHP+fD1TuYflJ3Ft0wmjH92tohpwBmxcIYB5zYqzX3juvHZ+tzuHv+6qAZg/Hl\nxhzGPPEFD36wjmO7teDj60dxw2nHEB1hlzUNdHbQ0BiHXPaHTmTlFvHC4s0kNY/hT6O7OR3piG3d\nV8z9/13DB6t20KlFDDNTkzmpVxunY5l6ZMXCGAfdfHovtuYW89AH6+gQH805A9s7HemwlJRVMOPL\nzTz92SYAbjy1J1eM6kpUuO1JBBsrFsY4KCREePTCgezML+HGN5bTJjYqYK769um6ndzz3hq27Cni\njH5tuf2s3iQmxDgdy/iInbMwxmFR4aG8ODGZxIRorpiTyU85hU5HqtOWPfuZMiuDybMyCQ0RXp4y\njOcuG2qFIshJsJxYS05O1szMTKdjGHPEft1TxLnPfk1MZCjzrhnpswv8qCoHyispraiktLzSdb/8\n4P2K/92vqORAWdV2Ffyyez+zv91CeIgw/eQepI3sQkSYfecMZCKyRFWTPbWzw1DGNIDKSqW0osoH\nc0UNH87llUz4Qyee/GQjyfcv4r5xfQkJkdo/0N0f5gcqan+upoJQepRddccObM9tZ/ambVxUPf12\nTCCwYmGCVvUP6No/SCt+94Fa9QPd1a7i0A/s371GxSEfxtVfv6zi8Pfg73x39SHLQgQiw0KJCAsh\nIiyESPe/EaGu+5FhocREhBF/cFm4698I93NV1/vduuEhRITW/LpRVZ6LiQy1WWEbKZ8WCxEZAzwB\nhAIzVPWhWtpdALwJpKhqpoiEAzOAIe6Mc1T1QV9mNf5l/Y4C3lqSdcgH+IFqH/p1fes+kg/omoSG\nSA0frCFEuD98I8NCaBoZRkTM7z+cf/cBXWW9yDqeO7jeaz/8yivf/cpZ/dvxwLn9fmsXFmqHfIwz\nfFYsRCQUeAY4FcgGMkRkvqquqdauGTAd+L7K4guBSFXtLyIxwBoReU1Vf/FVXuNfMrfsZcZXP3M4\np9TCQ4X28dEkJkTTIT6atnHRRIfX8G26lm/a1b+hH3zOict53v/H/kSEhjLz658Z0imBKcd1afAM\nxlTlyz2LYcAmVd0MICKvA+OANdXa3Qf8HbipyjIFmohIGBANlAL5Psxq/MyE4Z24ODmJvftL2VVw\ngJzCA+QU1HBzLy88UE5ZhbJlTxFb9hQBrkM2LZpG0qppJK1jXf+2ava/W7Oo8N/uN4kI9bupKG4/\nqzdb9xVx/4I1dIiPYky/dk5HMo2YL4tFByCryuNsYHjVBiIyGEhS1f+KSNVi8RauwrIdiAGuV9W9\nPsxq/FBYaAitY6NoHev5ROr+A+XsLvx9EdmV//uCsm57ATmFB6ioPHR3JTo8lFbNImldpZgcLC6u\nQhNFq2aRtGgaQXgDHQoKDRH+efFgLp3xHde+vozXroxiSMeEBvnZxlTny2JR09e03/5KRSQEeBxI\nraHdMKACaA8kAF+KyKKDeylVXuNK4EqAjh071k9qE5CaRIbRJDKMTi2a1NmuslLJLSr93Z7Krmp7\nKxt3FfL1pt3klxw6I6wINI+JOKSgVL21buYqLrHRYUe9txIdEcqMy5M577lvmDo7k3euHkHnlnVv\nozG+4LNxFiJyLPA3VT3d/fhWgIMnqkUkDvgJODgCqS2wFxgLpAHfqerL7rYzgQ9V9Y3afp6NszD1\nraSs4nd7K7uqHfqqequpO2pEWMghxaR1LUUmMqzu6TE25xRy3nPfkBATwdtXj6B5kwhfbbZpZLwd\nZ+HLYhEGbABOBrYCGcClqnpof0BX+8+Bm9y9oW4GegGTcR2GygDGq+qK2n6eFQvjFFUlv7icnMKS\nQ/ZSfndIrPAAe/eX1vgacdHhvz/01ezQPZZf9xQx7dWlDEiM45Wpw23+JVMvHB+Up6rlIjINWIir\n6+xMVV0tIvcCmao6v47VnwHSgVW4Dmel11UojHGSiBAXE05cTDjdWzers21ZRSV7Ckvdeyolh5yo\n31VwgGVZ+9hVUEJJWc2D5zK35PLAgrXc98d+vtgcY2rk03EWqvo+8H61ZXfV0vaEKvcLcXWfNSao\nhIeG0DYuyj36Oa7WdqrK/tIKVwHJLznkHMuQTvENF9oYbAS3MX5JRGgaGUbTyDC62Alt4wdsOKgx\nxhiPrFgYY4zxyIqFMcYYj6xYGGOM8ciKhTHGGI+sWBhjjPHIioUxxhiPrFgYY4zxyGdzQzU0EckB\nthzFS7QEdtdTHCcFy3aAbYu/CpZtCZbtgKPblk6q2spTo6ApFkdLRDK9mUzL3wXLdoBti78Klm0J\nlu2AhtkWOwxljDHGIysWxhhjPLJi8T8vOh2gngTLdoBti78Klm0Jlu2ABtgWO2dhjDHGI9uzMMYY\n41GjLRYicp+IrBCRZSLykYi0r6XdJBHZ6L5NauicnojIIyKyzr0t80SkxqviiMgvIrLSvb1+ef3Z\nw9iWMSKyXkQ2icgtDZ3TGyJyoYisFpFKEam1l0qAvC/ebotfvy8i0lxEPnb/LX8sIgm1tKtwvx/L\nRKSuK3o2OE+/YxGJFJG57ue/F5HO9fbDVbVR3oDYKvenA8/X0KY5sNn9b4L7foLT2atlPA0Ic99/\nGHi4lna/AC2dznu024LrEr0/AV2BCGA50Mfp7DXk7A0cA3wOJNfRLhDeF4/bEgjvC/B34Bb3/Vvq\n+FspdDrrkf6OgWsOfpYB44G59fXzG+2eharmV3nYBKjp5M3pwMequldVc4GPgTENkc9bqvqRqpa7\nH34HJDqZ52h4uS3DgE2qullVS4HXgXENldFbqrpWVdc7naM+eLktgfC+jANmu+/PBv7oYJYj4c3v\nuOo2vgWcLCJSHz+80RYLABF5QESygAlATdcG7wBkVXmc7V7mryYDH9TynAIficgSEbmyATMdqdq2\nJdDeE08C7X2pTSC8L21UdTuA+9/WtbSLEpFMEflORPypoHjzO/6tjfuLVx7Qoj5+eFBfg1tEFgFt\na3jqdlV9V1VvB24XkVuBacDd1V+ihnUbvPuYp+1wt7kdKAf+XcvLjFTVbSLSGvhYRNap6he+SVy7\netgWv3hPwLtt8ULAvC+eXqKGZX71t3IYL9PR/Z50BT4VkZWq+lP9JDwq3vyOffY+BHWxUNVTvGz6\nKrCAQ4tFNnBClceJuI7bNihP2+E+8X42cLK6D1bW8Brb3P/uEpF5uHZpG/xDqR62JRtIqvI4EdhW\nfwm9dxj/v+p6jYB4X7zgF+9LXdshIjtFpJ2qbheRdsCuWl7j4HuyWUQ+BwbjOlfgNG9+xwfbZItI\nGBAH7K2PH95oD0OJSI8qD8cC62pothA4TUQS3D0nTnMv8xsiMga4GRirqkW1tGkiIs0O3se1Hasa\nLqV3vNkWIAPoISJdRCQC10k8v+qx4q1AeV+8FAjvy3zgYI/GScAhe0zuv/VI9/2WwEhgTYMlrJs3\nv+Oq23gB8GltXyAPm9Nn+J26AW/j+sNcAbwHdHAvTwZmVGk3GdjkvqU5nbuG7diE6xjlMvftYE+I\n9sD77vtdcfWcWA6sxnVowfHsR7It7sdnAhtwfdvz1205F9e3vAPATmBhAL8vHrclEN4XXMfuPwE2\nuv9t7l7+2988MAJY6X5PVgJTnM5dbRsO+R0D9+L6ggUQBbzp/lv6AehaXz/bRnAbY4zxqNEehjLG\nGOM9KxbGGGM8smJhjDHGIysWxhhjPLJiYYwxxiMrFsYcBhEpPMr133KPDEZEmorICyLyk3tW1y9E\nZLiIRLjvB/WgWRNYrFgY00BEpC8Qqqqb3Ytm4Bpd20NV+wKpuGagLcU1DuBiR4IaUwMrFsYcAXF5\nRERWua9HcbF7eYiIPOveU/iviLwvIhe4V5uAe9SwiHQDhgN3qGoluKaXUNUF7rb/cbc3xi/Ybq4x\nR+Y8YBAwEGgJZIjIF7imh+gM9Mc1q+laYKZ7nZHAa+77fYFlqlpRy+uvAlJ8ktyYI2B7FsYcmeOA\n11S1QlV3AotxfbgfB7ypqpWqugP4rMo67YAcb17cXURKD84dZYzTrFgYc2Rqu6BMXReaKcY1dw+4\n5oIaKCJ1/Q1GAiVHkM2YemfFwpgj8wVwsYiEikgrYBSuidu+As53n7tow++nuF8LdAdQ1/URMoF7\nDl7JTER6iMg49/0WQI6qljXUBhlTFysWxhyZebhmLF4OfAr81X3Y6W1cM7SuAl4Avsd1tTJwXTPl\nhCqvMRXXhXo2ichK4CX+d32CE4H3fbsJxnjPZp01pp6JSFNVLXTvHfyA62p4O0QkGtc5jJF1nNg+\n+BrvALdqkFzH2wQ+6w1lTP37r4jEAxHAfe49DlS1WETuxnWd5F9rW9l9YZv/WKEw/sT2LIwxxnhk\n5yyMMcZ4ZMXCGGOMR1YsjDHGeGTFwhhjjEdWLIwxxnhkxcIYY4xH/x8MINTS1xQClwAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113c7b690>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "n_Cs = len(Cs)\n",
    "n_classes = 3\n",
    "scores = np.zeros((n_classes, n_Cs))\n",
    "\n",
    "for j in range(n_classes):\n",
    "    scores[j][:] = np.mean(lrcv_L2.scores_[j], axis = 0)\n",
    "\n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "pyplot.plot(np.log10(Cs), mse_mean.reshape(n_Cs, 1))\n",
    "pyplot.xlabel('log(C)')\n",
    "pyplot.ylabel('neg-logloss')\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C is: [ 0.1   0.05  0.01]\n"
     ]
    }
   ],
   "source": [
    "print 'C is:',lrcv_L2.C_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lrcv_L2 = LogisticRegressionCV(Cs=0.05, cv=5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L2.fit(X_train, y_train)\n",
    "pred = lrcv.predict(X_test)\n",
    "pred.to_csv('pred_lr.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
